From 7d9a626aaba9837cb82d189a9c4f0bcef58825bb Mon Sep 17 00:00:00 2001 From: Michael Tyler Date: Wed, 1 Feb 2023 16:37:07 +0000 Subject: Update CPU kernels to remove x19 and w19 Resolves: COMPMID-5805 Change-Id: Idf720bbb136474810086f5089c5ed23b3f79835a Signed-off-by: Michael Tyler Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/9081 Benchmark: Arm Jenkins Tested-by: Arm Jenkins Comments-Addressed: Arm Jenkins Reviewed-by: Gunes Bayir Reviewed-by: Viet-Hoa Do --- .../winograd/input_transforms/a64_fp32_6x6.cpp | 44 +++++------ .../winograd/input_transforms/sme_fp32_mla_6x6.cpp | 18 ++--- .../winograd/input_transforms/sve_fp32_6x6.cpp | 18 ++--- .../output_transforms/sme_fp32_mopa_4x4_3x3.cpp | 90 +++++++++++----------- 4 files changed, 85 insertions(+), 85 deletions(-) (limited to 'src/core/NEON/kernels/convolution') diff --git a/src/core/NEON/kernels/convolution/winograd/input_transforms/a64_fp32_6x6.cpp b/src/core/NEON/kernels/convolution/winograd/input_transforms/a64_fp32_6x6.cpp index 61741439e0..a2c04e0d8d 100644 --- a/src/core/NEON/kernels/convolution/winograd/input_transforms/a64_fp32_6x6.cpp +++ b/src/core/NEON/kernels/convolution/winograd/input_transforms/a64_fp32_6x6.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,9 +45,9 @@ void a64_fp32_6x6( "add x25, %[inptr0], %[input_row_stride]\n" "add x10, %[input_col_stride1], %[input_col_stride1]\n" "add x16, x25, %[input_row_stride]\n" - "add x19, x10, %[input_col_stride1]\n" + "add x8, x10, %[input_col_stride1]\n" "add x26, x16, %[input_row_stride]\n" - "add x20, x19, %[input_col_stride1]\n" + "add x20, x8, %[input_col_stride1]\n" "add x17, x26, %[input_row_stride]\n" "add x21, x20, %[input_col_stride1]\n" "add x27, x17, %[input_row_stride]\n" @@ -70,7 +70,7 @@ void a64_fp32_6x6( "mov v10.16b, v8.16b\n" "ldr q1, [%[inptr0], x21]\n" "fmla v14.4s, v9.4s, v0.s[2]\n" - "ldr q4, [%[inptr0], x19]\n" + "ldr q4, [%[inptr0], x8]\n" "mov v9.16b, v8.16b\n" "ldr q12, [%[inptr0], %[input_col_stride1]]\n" "fmls v10.4s, v12.4s, v0.s[2]\n" @@ -82,7 +82,7 @@ void a64_fp32_6x6( "fmls v10.4s, v2.4s, v0.s[2]\n" "ldr q6, [x16, x21]\n" "mov v7.16b, v8.16b\n" - "ldr q16, [x16, x19]\n" + "ldr q16, [x16, x8]\n" "fmls v9.4s, v2.4s, v0.s[2]\n" "ldr q22, [x16, %[input_col_stride1]]\n" "fadd v10.4s, v10.4s, v4.4s\n" @@ -94,7 +94,7 @@ void a64_fp32_6x6( "mov v8.16b, v8.16b\n" "ldr q18, [x17, x21]\n" "fsub v7.4s, v7.4s, v2.4s\n" - "ldr q13, [x17, x19]\n" + "ldr q13, [x17, x8]\n" "fmla v7.4s, v4.4s, v0.s[1]\n" "ldr q21, [x17, %[input_col_stride1]]\n" "fmla v8.4s, v12.4s, v0.s[1]\n" @@ -185,7 +185,7 @@ void a64_fp32_6x6( "fmla v10.4s, v7.4s, v0.s[2]\n" "ldr q13, [x25, x21]\n" "mov v7.16b, v11.16b\n" - "ldr q31, [x25, x19]\n" + "ldr q31, [x25, x8]\n" "mov v8.16b, v11.16b\n" "ldr q21, [x25, %[input_col_stride1]]\n" "fmls v10.4s, v23.4s, v0.s[3]\n" @@ -197,7 +197,7 @@ void a64_fp32_6x6( "fmls v8.4s, v21.4s, v0.s[1]\n" "ldr q24, [x26, x21]\n" "fmls v9.4s, v23.4s, v0.s[2]\n" - "ldr q27, [x26, x19]\n" + "ldr q27, [x26, x8]\n" "fmls v7.4s, v23.4s, v0.s[2]\n" "ldr q28, [x26, %[input_col_stride1]]\n" "fsub v8.4s, v8.4s, v23.4s\n" @@ -365,7 +365,7 @@ void a64_fp32_6x6( "fmla v4.4s, v18.4s, v0.s[2]\n" "ldr q3, [x27, x21]\n" "mov v6.16b, v2.16b\n" - "ldr q5, [x27, x19]\n" + "ldr q5, [x27, x8]\n" "mov v1.16b, v2.16b\n" "ldr q18, [x27, %[input_col_stride1]]\n" "fmls v4.4s, v17.4s, v0.s[3]\n" @@ -425,7 +425,7 @@ void a64_fp32_6x6( "fmla v14.4s, v9.4s, v0.s[2]\n" "ldr d1, [%[inptr0], x21]\n" "mov v9.16b, v8.16b\n" - "ldr d4, [%[inptr0], x19]\n" + "ldr d4, [%[inptr0], x8]\n" "mov v7.16b, v8.16b\n" "ldr d12, [%[inptr0], %[input_col_stride1]]\n" "fmls v14.4s, v2.4s, v0.s[3]\n" @@ -437,7 +437,7 @@ void a64_fp32_6x6( "fmls v7.4s, v12.4s, v0.s[1]\n" "ldr d6, [x16, x21]\n" "fmls v10.4s, v2.4s, v0.s[2]\n" - "ldr d16, [x16, x19]\n" + "ldr d16, [x16, x8]\n" "fmls v9.4s, v2.4s, v0.s[2]\n" "ldr d22, [x16, %[input_col_stride1]]\n" "fsub v7.4s, v7.4s, v2.4s\n" @@ -449,7 +449,7 @@ void a64_fp32_6x6( "fmla v7.4s, v4.4s, v0.s[1]\n" "ldr d18, [x17, x21]\n" "mov v8.16b, v8.16b\n" - "ldr d13, [x17, x19]\n" + "ldr d13, [x17, x8]\n" "mov v11.16b, v1.16b\n" "ldr d21, [x17, %[input_col_stride1]]\n" "fmla v8.4s, v12.4s, v0.s[1]\n" @@ -539,7 +539,7 @@ void a64_fp32_6x6( "fmla v10.4s, v7.4s, v0.s[2]\n" "ldr d13, [x25, x21]\n" "mov v7.16b, v11.16b\n" - "ldr d31, [x25, x19]\n" + "ldr d31, [x25, x8]\n" "mov v8.16b, v11.16b\n" "ldr d21, [x25, %[input_col_stride1]]\n" "fmls v10.4s, v23.4s, v0.s[3]\n" @@ -551,7 +551,7 @@ void a64_fp32_6x6( "fmls v8.4s, v21.4s, v0.s[1]\n" "ldr d24, [x26, x21]\n" "fmls v9.4s, v23.4s, v0.s[2]\n" - "ldr d27, [x26, x19]\n" + "ldr d27, [x26, x8]\n" "fmls v7.4s, v23.4s, v0.s[2]\n" "ldr d28, [x26, %[input_col_stride1]]\n" "fsub v8.4s, v8.4s, v23.4s\n" @@ -719,7 +719,7 @@ void a64_fp32_6x6( "fmla v4.4s, v18.4s, v0.s[2]\n" "ldr d3, [x27, x21]\n" "mov v6.16b, v2.16b\n" - "ldr d5, [x27, x19]\n" + "ldr d5, [x27, x8]\n" "mov v1.16b, v2.16b\n" "ldr d18, [x27, %[input_col_stride1]]\n" "fmls v4.4s, v17.4s, v0.s[3]\n" @@ -776,7 +776,7 @@ void a64_fp32_6x6( "fmla v14.4s, v9.4s, v0.s[2]\n" "ldr s1, [%[inptr0], x21]\n" "mov v9.16b, v8.16b\n" - "ldr s4, [%[inptr0], x19]\n" + "ldr s4, [%[inptr0], x8]\n" "mov v7.16b, v8.16b\n" "ldr s12, [%[inptr0], %[input_col_stride1]]\n" "fmls v14.4s, v2.4s, v0.s[3]\n" @@ -788,7 +788,7 @@ void a64_fp32_6x6( "fmls v7.4s, v12.4s, v0.s[1]\n" "ldr s6, [x16, x21]\n" "fmls v10.4s, v2.4s, v0.s[2]\n" - "ldr s16, [x16, x19]\n" + "ldr s16, [x16, x8]\n" "fmls v9.4s, v2.4s, v0.s[2]\n" "ldr s22, [x16, %[input_col_stride1]]\n" "fsub v7.4s, v7.4s, v2.4s\n" @@ -800,7 +800,7 @@ void a64_fp32_6x6( "fmla v7.4s, v4.4s, v0.s[1]\n" "ldr s18, [x17, x21]\n" "mov v8.16b, v8.16b\n" - "ldr s13, [x17, x19]\n" + "ldr s13, [x17, x8]\n" "mov v11.16b, v1.16b\n" "ldr s21, [x17, %[input_col_stride1]]\n" "fmla v8.4s, v12.4s, v0.s[1]\n" @@ -890,7 +890,7 @@ void a64_fp32_6x6( "fmla v10.4s, v7.4s, v0.s[2]\n" "ldr s13, [x25, x21]\n" "mov v7.16b, v11.16b\n" - "ldr s31, [x25, x19]\n" + "ldr s31, [x25, x8]\n" "mov v8.16b, v11.16b\n" "ldr s21, [x25, %[input_col_stride1]]\n" "fmls v10.4s, v23.4s, v0.s[3]\n" @@ -902,7 +902,7 @@ void a64_fp32_6x6( "fmls v8.4s, v21.4s, v0.s[1]\n" "ldr s24, [x26, x21]\n" "fmls v9.4s, v23.4s, v0.s[2]\n" - "ldr s27, [x26, x19]\n" + "ldr s27, [x26, x8]\n" "fmls v7.4s, v23.4s, v0.s[2]\n" "ldr s28, [x26, %[input_col_stride1]]\n" "fsub v8.4s, v8.4s, v23.4s\n" @@ -1070,7 +1070,7 @@ void a64_fp32_6x6( "fmla v4.4s, v18.4s, v0.s[2]\n" "ldr s3, [x27, x21]\n" "mov v6.16b, v2.16b\n" - "ldr s5, [x27, x19]\n" + "ldr s5, [x27, x8]\n" "mov v1.16b, v2.16b\n" "ldr s18, [x27, %[input_col_stride1]]\n" "fmls v4.4s, v17.4s, v0.s[3]\n" @@ -1128,7 +1128,7 @@ void a64_fp32_6x6( : "cc", "v0", "v1", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v2", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v3", "v30", "v31", "v4", "v5", "v6", "v7", "v8", - "v9", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x10", "x19", + "v9", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x10", "x8", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "memory" ); } diff --git a/src/core/NEON/kernels/convolution/winograd/input_transforms/sme_fp32_mla_6x6.cpp b/src/core/NEON/kernels/convolution/winograd/input_transforms/sme_fp32_mla_6x6.cpp index e2397cd750..f446e7ea8b 100644 --- a/src/core/NEON/kernels/convolution/winograd/input_transforms/sme_fp32_mla_6x6.cpp +++ b/src/core/NEON/kernels/convolution/winograd/input_transforms/sme_fp32_mla_6x6.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -65,7 +65,7 @@ void sme_fp32_mla_6x6( "lsl x22, %x[input_col_1_stride], #0x2\n" "lsl x21, %x[output_col_1_stride], #0x2\n" "add x20, x22, %x[input_col_1_stride]\n" - "add x19, x21, %x[output_col_1_stride]\n" + "add x8, x21, %x[output_col_1_stride]\n" "whilelt p0.s, XZR, %x[num_channels]\n" "beq 2f\n" "1:" // channel_loop @@ -156,7 +156,7 @@ void sme_fp32_mla_6x6( "st1w { z27.s }, p0, [%x[output_row_0], x25, LSL #2]\n" "st1w { z8.s }, p0, [%x[output_row_0], x23, LSL #2]\n" "st1w { z25.s }, p0, [%x[output_row_0], x21, LSL #2]\n" - "st1w { z28.s }, p0, [%x[output_row_0], x19, LSL #2]\n" + "st1w { z28.s }, p0, [%x[output_row_0], x8, LSL #2]\n" "incb %x[output_row_0]\n" "ld1w { z19.s }, p0/Z, [x16]\n" "ld1w { z7.s }, p0/Z, [x16, %x[input_col_1_stride], LSL #2]\n" @@ -283,10 +283,10 @@ void sme_fp32_mla_6x6( "fneg z10.s, p1/M, z10.s\n" "fadd z10.s, z10.s, z31.s\n" "fadd z17.s, z13.s, z15.s\n" - "st1w { z17.s }, p0, [x11, x19, LSL #2]\n" + "st1w { z17.s }, p0, [x11, x8, LSL #2]\n" "fsub z17.s, z15.s, z13.s\n" "incb x11\n" - "st1w { z17.s }, p0, [x9, x19, LSL #2]\n" + "st1w { z17.s }, p0, [x9, x8, LSL #2]\n" "fadd z17.s, z10.s, z14.s\n" "fsub z14.s, z14.s, z10.s\n" "st1w { z22.s }, p0, [x15]\n" @@ -300,9 +300,9 @@ void sme_fp32_mla_6x6( "st1w { z4.s }, p0, [x13, x23, LSL #2]\n" "st1w { z18.s }, p0, [x15, x21, LSL #2]\n" "st1w { z30.s }, p0, [x13, x21, LSL #2]\n" - "st1w { z17.s }, p0, [x15, x19, LSL #2]\n" + "st1w { z17.s }, p0, [x15, x8, LSL #2]\n" "incb x15\n" - "st1w { z14.s }, p0, [x13, x19, LSL #2]\n" + "st1w { z14.s }, p0, [x13, x8, LSL #2]\n" "incb x13\n" "ld1w { z23.s }, p0/Z, [x28]\n" "ld1w { z22.s }, p0/Z, [x28, %x[input_col_1_stride], LSL #2]\n" @@ -344,7 +344,7 @@ void sme_fp32_mla_6x6( "st1w { z6.s }, p0, [x27, x25, LSL #2]\n" "st1w { z28.s }, p0, [x27, x23, LSL #2]\n" "st1w { z8.s }, p0, [x27, x21, LSL #2]\n" - "st1w { z7.s }, p0, [x27, x19, LSL #2]\n" + "st1w { z7.s }, p0, [x27, x8, LSL #2]\n" "incb x27\n" "whilelt p0.s, XZR, %x[num_channels]\n" "bne 1b\n" @@ -352,7 +352,7 @@ void sme_fp32_mla_6x6( ".inst 0xd503467f // SMSTOP\n" : [input_row_0] "+&r" (input), [num_channels] "+&r" (long_channels), [output_row_0] "+&r" (output) : [B_values] "r" (B_values), [input_col_1_stride] "r" ((long) input_col_stride), [input_row_stride] "r" ((long) input_row_stride), [output_col_1_stride] "r" ((long) output_col_stride), [output_row_stride] "r" (6 * (long) output_col_stride) - : "cc", "memory", "p0", "p1", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x8", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/convolution/winograd/input_transforms/sve_fp32_6x6.cpp b/src/core/NEON/kernels/convolution/winograd/input_transforms/sve_fp32_6x6.cpp index cb5ece05d6..7b387e1247 100644 --- a/src/core/NEON/kernels/convolution/winograd/input_transforms/sve_fp32_6x6.cpp +++ b/src/core/NEON/kernels/convolution/winograd/input_transforms/sve_fp32_6x6.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -63,7 +63,7 @@ void sve_fp32_6x6( "lsl x22, %x[input_col_1_stride], #0x2\n" "lsl x21, %x[output_col_1_stride], #0x2\n" "add x20, x22, %x[input_col_1_stride]\n" - "add x19, x21, %x[output_col_1_stride]\n" + "add x8, x21, %x[output_col_1_stride]\n" "whilelt p0.s, XZR, %x[num_channels]\n" "beq 2f\n" "1:" // channel_loop @@ -154,7 +154,7 @@ void sve_fp32_6x6( "st1w { z27.s }, p0, [%x[output_row_0], x25, LSL #2]\n" "st1w { z8.s }, p0, [%x[output_row_0], x23, LSL #2]\n" "st1w { z25.s }, p0, [%x[output_row_0], x21, LSL #2]\n" - "st1w { z28.s }, p0, [%x[output_row_0], x19, LSL #2]\n" + "st1w { z28.s }, p0, [%x[output_row_0], x8, LSL #2]\n" "incb %x[output_row_0]\n" "ld1w { z19.s }, p0/Z, [x16]\n" "ld1w { z7.s }, p0/Z, [x16, %x[input_col_1_stride], LSL #2]\n" @@ -281,10 +281,10 @@ void sve_fp32_6x6( "fneg z10.s, p1/M, z10.s\n" "fadd z10.s, z10.s, z31.s\n" "fadd z17.s, z13.s, z15.s\n" - "st1w { z17.s }, p0, [x11, x19, LSL #2]\n" + "st1w { z17.s }, p0, [x11, x8, LSL #2]\n" "fsub z17.s, z15.s, z13.s\n" "incb x11\n" - "st1w { z17.s }, p0, [x9, x19, LSL #2]\n" + "st1w { z17.s }, p0, [x9, x8, LSL #2]\n" "fadd z17.s, z10.s, z14.s\n" "fsub z14.s, z14.s, z10.s\n" "st1w { z22.s }, p0, [x15]\n" @@ -298,9 +298,9 @@ void sve_fp32_6x6( "st1w { z4.s }, p0, [x13, x23, LSL #2]\n" "st1w { z18.s }, p0, [x15, x21, LSL #2]\n" "st1w { z30.s }, p0, [x13, x21, LSL #2]\n" - "st1w { z17.s }, p0, [x15, x19, LSL #2]\n" + "st1w { z17.s }, p0, [x15, x8, LSL #2]\n" "incb x15\n" - "st1w { z14.s }, p0, [x13, x19, LSL #2]\n" + "st1w { z14.s }, p0, [x13, x8, LSL #2]\n" "incb x13\n" "ld1w { z23.s }, p0/Z, [x28]\n" "ld1w { z22.s }, p0/Z, [x28, %x[input_col_1_stride], LSL #2]\n" @@ -342,7 +342,7 @@ void sve_fp32_6x6( "st1w { z6.s }, p0, [x27, x25, LSL #2]\n" "st1w { z28.s }, p0, [x27, x23, LSL #2]\n" "st1w { z8.s }, p0, [x27, x21, LSL #2]\n" - "st1w { z7.s }, p0, [x27, x19, LSL #2]\n" + "st1w { z7.s }, p0, [x27, x8, LSL #2]\n" "incb x27\n" "whilelt p0.s, XZR, %x[num_channels]\n" "bne 1b\n" @@ -350,7 +350,7 @@ void sve_fp32_6x6( : [input_row_0] "+&r" (input), [num_channels] "+&r" (long_channels), [output_row_0] "+&r" (output) : [B_values] "r" (B_values), [input_col_1_stride] "r" ((long) input_col_stride), [input_row_stride] "r" ((long) input_row_stride), [output_col_1_stride] "r" ((long) output_col_stride), [output_row_stride] "r" (6 * (long) output_col_stride) - : "cc", "memory", "p0", "p1", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x8", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp index 9d3c751d78..54749e6f28 100644 --- a/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright (c) 2022-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -112,17 +112,17 @@ void sme_fp32_mopa_4x4_3x3( "ld1rw { z12.s }, p5/Z, [%x[params], %[offsetof_Params_act_min]]\n" "ld1rw { z10.s }, p5/Z, [%x[params], %[offsetof_Params_act_max]]\n" "pfalse p8.b\n" - "ldr x19, [%x[params], %[offsetof_Params_inner_terms]]\n" + "ldr x8, [%x[params], %[offsetof_Params_inner_terms]]\n" "ld1w { z6.s }, p5/Z, [x20]\n" "ld1w { z7.s }, p5/Z, [x20, #1, MUL VL]\n" - "ld1rqw { z9.s }, p5/Z, [x19]\n" - "ld1rqw { z8.s }, p5/Z, [x19, #16]\n" - "ld1rqw { z15.s }, p5/Z, [x19, #32]\n" + "ld1rqw { z9.s }, p5/Z, [x8]\n" + "ld1rqw { z8.s }, p5/Z, [x8, #16]\n" + "ld1rqw { z15.s }, p5/Z, [x8, #32]\n" "fmul z11.s, z9.s, z6.s[0]\n" "fmul z5.s, z9.s, z6.s[1]\n" - "ld1rqw { z4.s }, p5/Z, [x19, #48]\n" - "ld1rqw { z3.s }, p5/Z, [x19, #64]\n" - "ld1rqw { z2.s }, p5/Z, [x19, #80]\n" + "ld1rqw { z4.s }, p5/Z, [x8, #48]\n" + "ld1rqw { z3.s }, p5/Z, [x8, #64]\n" + "ld1rqw { z2.s }, p5/Z, [x8, #80]\n" "cbz %x[bptr], 1f\n" "ptrue p8.s\n" "1:" // Set bias predicate: Done @@ -177,7 +177,7 @@ void sme_fp32_mopa_4x4_3x3( "add x20, %x[output], %x[output_col_stride], LSL #2\n" ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" - "add x19, %x[output], %x[output_row_stride], LSL #2\n" + "add x8, %x[output], %x[output_row_stride], LSL #2\n" ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" "fmul z11.s, z9.s, z6.s[2]\n" "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" @@ -567,25 +567,25 @@ void sme_fp32_mopa_4x4_3x3( "add x20, x20, %x[output_col_stride], LSL #2\n" "fmax z17.s, p5/M, z17.s, z12.s\n" "st1w { z28.s }, p0, [x20, x25, LSL #2]\n" - "add x20, x19, %x[output_col_stride], LSL #2\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" "fmax z16.s, p5/M, z16.s, z12.s\n" - "st1w { z27.s }, p0, [x19, x25, LSL #2]\n" - "add x19, x19, %x[output_row_stride], LSL #2\n" + "st1w { z27.s }, p0, [x8, x25, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z26.s }, p0, [x20, x25, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z25.s }, p0, [x20, x25, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z24.s }, p0, [x20, x25, LSL #2]\n" - "add x20, x19, %x[output_col_stride], LSL #2\n" - "st1w { z23.s }, p0, [x19, x25, LSL #2]\n" - "add x19, x19, %x[output_row_stride], LSL #2\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z23.s }, p0, [x8, x25, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z22.s }, p0, [x20, x25, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z21.s }, p0, [x20, x25, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z20.s }, p0, [x20, x25, LSL #2]\n" - "add x20, x19, %x[output_col_stride], LSL #2\n" - "st1w { z19.s }, p0, [x19, x25, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z19.s }, p0, [x8, x25, LSL #2]\n" "st1w { z18.s }, p0, [x20, x25, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z17.s }, p0, [x20, x25, LSL #2]\n" @@ -624,7 +624,7 @@ void sme_fp32_mopa_4x4_3x3( ".inst 0xc08254d4 // mova z20.s, p5/M, za1h.s[x14, #2]\n" "fmax z25.s, p5/M, z25.s, z12.s\n" "fmin z21.s, p5/M, z21.s, z10.s\n" - "add x19, %x[output], %x[output_row_stride], LSL #2\n" + "add x8, %x[output], %x[output_row_stride], LSL #2\n" ".inst 0xc08274f3 // mova z19.s, p5/M, za1h.s[XZR, #3]\n" "fmax z24.s, p5/M, z24.s, z12.s\n" "fmin z20.s, p5/M, z20.s, z10.s\n" @@ -648,25 +648,25 @@ void sme_fp32_mopa_4x4_3x3( "add x20, x20, %x[output_col_stride], LSL #2\n" "fmax z17.s, p5/M, z17.s, z12.s\n" "st1w { z28.s }, p0, [x20, x24, LSL #2]\n" - "add x20, x19, %x[output_col_stride], LSL #2\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" "fmax z16.s, p5/M, z16.s, z12.s\n" - "st1w { z27.s }, p0, [x19, x24, LSL #2]\n" - "add x19, x19, %x[output_row_stride], LSL #2\n" + "st1w { z27.s }, p0, [x8, x24, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z26.s }, p0, [x20, x24, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z25.s }, p0, [x20, x24, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z24.s }, p0, [x20, x24, LSL #2]\n" - "add x20, x19, %x[output_col_stride], LSL #2\n" - "st1w { z23.s }, p0, [x19, x24, LSL #2]\n" - "add x19, x19, %x[output_row_stride], LSL #2\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z23.s }, p0, [x8, x24, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z22.s }, p0, [x20, x24, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z21.s }, p0, [x20, x24, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z20.s }, p0, [x20, x24, LSL #2]\n" - "add x20, x19, %x[output_col_stride], LSL #2\n" - "st1w { z19.s }, p0, [x19, x24, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z19.s }, p0, [x8, x24, LSL #2]\n" "st1w { z18.s }, p0, [x20, x24, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z17.s }, p0, [x20, x24, LSL #2]\n" @@ -705,7 +705,7 @@ void sme_fp32_mopa_4x4_3x3( ".inst 0xc0825554 // mova z20.s, p5/M, za2h.s[x14, #2]\n" "fmax z25.s, p5/M, z25.s, z12.s\n" "fmin z21.s, p5/M, z21.s, z10.s\n" - "add x19, %x[output], %x[output_row_stride], LSL #2\n" + "add x8, %x[output], %x[output_row_stride], LSL #2\n" ".inst 0xc0827573 // mova z19.s, p5/M, za2h.s[XZR, #3]\n" "fmax z24.s, p5/M, z24.s, z12.s\n" "fmin z20.s, p5/M, z20.s, z10.s\n" @@ -729,25 +729,25 @@ void sme_fp32_mopa_4x4_3x3( "add x20, x20, %x[output_col_stride], LSL #2\n" "fmax z17.s, p5/M, z17.s, z12.s\n" "st1w { z28.s }, p0, [x20, x23, LSL #2]\n" - "add x20, x19, %x[output_col_stride], LSL #2\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" "fmax z16.s, p5/M, z16.s, z12.s\n" - "st1w { z27.s }, p0, [x19, x23, LSL #2]\n" - "add x19, x19, %x[output_row_stride], LSL #2\n" + "st1w { z27.s }, p0, [x8, x23, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z26.s }, p0, [x20, x23, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z25.s }, p0, [x20, x23, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z24.s }, p0, [x20, x23, LSL #2]\n" - "add x20, x19, %x[output_col_stride], LSL #2\n" - "st1w { z23.s }, p0, [x19, x23, LSL #2]\n" - "add x19, x19, %x[output_row_stride], LSL #2\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z23.s }, p0, [x8, x23, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z22.s }, p0, [x20, x23, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z21.s }, p0, [x20, x23, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z20.s }, p0, [x20, x23, LSL #2]\n" - "add x20, x19, %x[output_col_stride], LSL #2\n" - "st1w { z19.s }, p0, [x19, x23, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z19.s }, p0, [x8, x23, LSL #2]\n" "st1w { z18.s }, p0, [x20, x23, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z17.s }, p0, [x20, x23, LSL #2]\n" @@ -790,7 +790,7 @@ void sme_fp32_mopa_4x4_3x3( ".inst 0xc08275f3 // mova z19.s, p5/M, za3h.s[XZR, #3]\n" "fmax z24.s, p5/M, z24.s, z12.s\n" "fmin z20.s, p5/M, z20.s, z10.s\n" - "add x19, %x[output], %x[output_row_stride], LSL #2\n" + "add x8, %x[output], %x[output_row_stride], LSL #2\n" ".inst 0xc08235f2 // mova z18.s, p5/M, za3h.s[x13, #3]\n" "fmax z23.s, p5/M, z23.s, z12.s\n" "fmin z19.s, p5/M, z19.s, z10.s\n" @@ -817,24 +817,24 @@ void sme_fp32_mopa_4x4_3x3( "add x20, x20, %x[output_col_stride], LSL #2\n" "fmax z16.s, p5/M, z16.s, z12.s\n" "st1w { z28.s }, p0, [x20, x22, LSL #2]\n" - "add x20, x19, %x[output_col_stride], LSL #2\n" - "st1w { z27.s }, p0, [x19, x22, LSL #2]\n" - "add x19, x19, %x[output_row_stride], LSL #2\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z27.s }, p0, [x8, x22, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z26.s }, p0, [x20, x22, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z25.s }, p0, [x20, x22, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z24.s }, p0, [x20, x22, LSL #2]\n" - "add x20, x19, %x[output_col_stride], LSL #2\n" - "st1w { z23.s }, p0, [x19, x22, LSL #2]\n" - "add x19, x19, %x[output_row_stride], LSL #2\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z23.s }, p0, [x8, x22, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z22.s }, p0, [x20, x22, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z21.s }, p0, [x20, x22, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z20.s }, p0, [x20, x22, LSL #2]\n" - "add x20, x19, %x[output_col_stride], LSL #2\n" - "st1w { z19.s }, p0, [x19, x22, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z19.s }, p0, [x8, x22, LSL #2]\n" "st1w { z18.s }, p0, [x20, x22, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z17.s }, p0, [x20, x22, LSL #2]\n" @@ -880,7 +880,7 @@ void sme_fp32_mopa_4x4_3x3( ".inst 0xd503467f // SMSTOP\n" : : [bptr] "r" (bptr), [inptr] "r" (inptr), [matrix_stride] "r" (matrix_stride), [n_channels] "r" (n_channels), [offsetof_Params_act_max] "I" (offsetof(Params, act_max)), [offsetof_Params_act_min] "I" (offsetof(Params, act_min)), [offsetof_Params_inner_terms] "I" (offsetof(Params, inner_terms)), [offsetof_Params_outer_terms] "I" (offsetof(Params, outer_terms)), [output] "r" (output), [output_col_stride] "r" (output_col_stride), [output_row_stride] "r" (output_row_stride), [params] "r" (¶ms) - : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p8", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p8", "x12", "x13", "x14", "x8", "x20", "x21", "x22", "x23", "x24", "x25", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } -- cgit v1.2.1