diff options
Diffstat (limited to 'src/core/NEON/kernels/convolution/depthwise/depthwise_4x4_3x3_1x1_fp32_fp32.cpp')
| -rw-r--r-- | src/core/NEON/kernels/convolution/depthwise/depthwise_4x4_3x3_1x1_fp32_fp32.cpp | 2366 |
1 files changed, 2366 insertions, 0 deletions
diff --git a/src/core/NEON/kernels/convolution/depthwise/depthwise_4x4_3x3_1x1_fp32_fp32.cpp b/src/core/NEON/kernels/convolution/depthwise/depthwise_4x4_3x3_1x1_fp32_fp32.cpp index ff0e454c76..a583615c99 100644 --- a/src/core/NEON/kernels/convolution/depthwise/depthwise_4x4_3x3_1x1_fp32_fp32.cpp +++ b/src/core/NEON/kernels/convolution/depthwise/depthwise_4x4_3x3_1x1_fp32_fp32.cpp @@ -1169,6 +1169,1155 @@ void Conv::execute_tile<ActivationFunction::None>( template <> template <> +void Conv::execute_tile<ActivationFunction::None>( + int n_channels, + const void *weight_bias_ptr, + const float *inptrs[6][6], + float *outptrs[4][4] +) +{ + __asm __volatile( + "mov x27, xzr\n" + "mov x28, xzr\n" + "and x15, %[n_channels], #3\n" + "lsr x16, %[n_channels], #2\n" + "cbz x16, 4f\n" + "1:\n" + "ldr q13, [%[wbptr]]\n" + "ldr x17, [%[inptrs], 0]\n" + "mov v18.16b, v13.16b\n" + "ldr q12, [%[wbptr], #16]\n" + "mov v22.16b, v13.16b\n" + "ldr q11, [%[wbptr], #32]\n" + "mov v23.16b, v13.16b\n" + "ldr q10, [%[wbptr], #48]\n" + "mov v19.16b, v13.16b\n" + "ldr q9, [%[wbptr], #64]\n" + "mov v17.16b, v13.16b\n" + "ldr q8, [%[wbptr], #80]\n" + "mov v14.16b, v13.16b\n" + "ldr q7, [%[wbptr], #96]\n" + "mov v0.16b, v13.16b\n" + "ldr q6, [%[wbptr], #112]\n" + "mov v1.16b, v13.16b\n" + "ldr q5, [%[wbptr], #128]\n" + "mov v2.16b, v13.16b\n" + "ldr q4, [%[wbptr], #144]\n" + "ldr q29, [x17, x27]\n" + "ldr x18, [%[inptrs], 48]\n" + "fmla v18.4s, v29.4s, v12.4s\n" + "ldr x17, [%[inptrs], 8]\n" + "ldr q27, [x18, x27]\n" + "ldr x19, [%[inptrs], 96]\n" + "ldr q28, [x17, x27]\n" + "ldr x18, [%[inptrs], 56]\n" + "ldr q25, [x19, x27]\n" + "ldr x17, [%[inptrs], 16]\n" + "ldr q16, [x18, x27]\n" + "ldr x20, [%[inptrs], 144]\n" + "ldr q15, [x17, x27]\n" + "ldr x19, [%[inptrs], 104]\n" + "ldr q21, [x20, x27]\n" + "subs x16, x16, #1\n" + "ldr q29, [x19, x27]\n" + "beq 3f\n" + "2:\n" + "mov v3.16b, v13.16b\n" + "ldr x18, [%[inptrs], 64]\n" + "fmla v18.4s, v27.4s, v9.4s\n" + "ldr x17, [%[inptrs], 24]\n" + "fmla v22.4s, v27.4s, v12.4s\n" + "ldr q30, [x18, x27]\n" + "fmla v23.4s, v28.4s, v12.4s\n" + "ldr x21, [%[inptrs], 192]\n" + "fmla v19.4s, v25.4s, v12.4s\n" + "ldr x20, [%[inptrs], 152]\n" + "fmla v18.4s, v28.4s, v11.4s\n" + "ldr q24, [x17, x27]\n" + "fmla v22.4s, v25.4s, v9.4s\n" + "ldr x19, [%[inptrs], 112]\n" + "fmla v23.4s, v16.4s, v9.4s\n" + "ldr x18, [%[inptrs], 72]\n" + "fmla v17.4s, v16.4s, v12.4s\n" + "ldr x17, [%[inptrs], 32]\n" + "fmla v18.4s, v25.4s, v6.4s\n" + "ldr q31, [x21, x27]\n" + "fmla v22.4s, v16.4s, v11.4s\n" + "ldr x22, [%[inptrs], 240]\n" + "fmla v23.4s, v15.4s, v11.4s\n" + "ldr x21, [%[inptrs], 200]\n" + "fmla v14.4s, v15.4s, v12.4s\n" + "ldr x23, [%[outptrs], 0]\n" + "fmla v18.4s, v16.4s, v8.4s\n" + "ldr q25, [x20, x27]\n" + "fmla v22.4s, v21.4s, v6.4s\n" + "ldr x20, [%[inptrs], 160]\n" + "fmla v19.4s, v21.4s, v9.4s\n" + "ldr x24, [%[outptrs], 32]\n" + "fmla v0.4s, v21.4s, v12.4s\n" + "ldr q21, [x19, x27]\n" + "fmla v18.4s, v15.4s, v10.4s\n" + "ldr q20, [x18, x27]\n" + "fmla v22.4s, v29.4s, v8.4s\n" + "ldr x19, [%[inptrs], 120]\n" + "fmla v23.4s, v29.4s, v6.4s\n" + "ldr x18, [%[inptrs], 80]\n" + "fmla v19.4s, v29.4s, v11.4s\n" + "ldr x25, [%[outptrs], 64]\n" + "fmla v18.4s, v29.4s, v5.4s\n" + "ldr x26, [%[outptrs], 96]\n" + "fmla v17.4s, v29.4s, v9.4s\n" + "add %[wbptr], %[wbptr], #160\n" + "fmla v1.4s, v29.4s, v12.4s\n" + "ldr q26, [x17, x27]\n" + "fmla v22.4s, v30.4s, v10.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v18.4s, v30.4s, v7.4s\n" + "ldr x17, [%[inptrs], 40]\n" + "fmla v23.4s, v30.4s, v8.4s\n" + "subs x16, x16, #1\n" + "fmla v17.4s, v30.4s, v11.4s\n" + "fmla v14.4s, v30.4s, v9.4s\n" + "fmla v2.4s, v30.4s, v12.4s\n" + "ldr q27, [x22, x27]\n" + "fmla v3.4s, v24.4s, v12.4s\n" + "ldr x22, [%[inptrs], 248]\n" + "fmla v23.4s, v24.4s, v10.4s\n" + "fmla v19.4s, v31.4s, v6.4s\n" + "fmla v14.4s, v24.4s, v11.4s\n" + "ldr q30, [x21, x27]\n" + "fmla v0.4s, v31.4s, v9.4s\n" + "ldr q24, [x20, x27]\n" + "fmla v22.4s, v25.4s, v5.4s\n" + "ldr x21, [%[inptrs], 208]\n" + "fmla v19.4s, v25.4s, v8.4s\n" + "ldr x20, [%[inptrs], 168]\n" + "fmla v17.4s, v25.4s, v6.4s\n" + "fmla v1.4s, v25.4s, v9.4s\n" + "fmla v0.4s, v25.4s, v11.4s\n" + "fmla v18.4s, v21.4s, v4.4s\n" + "fmla v22.4s, v21.4s, v7.4s\n" + "fmla v23.4s, v21.4s, v5.4s\n" + "fmla v19.4s, v21.4s, v10.4s\n" + "fmla v14.4s, v21.4s, v6.4s\n" + "fmla v17.4s, v21.4s, v8.4s\n" + "fmla v1.4s, v21.4s, v11.4s\n" + "str q18, [x23, x28]\n" + "mov v16.16b, v13.16b\n" + "fmla v2.4s, v21.4s, v9.4s\n" + "ldr x23, [%[outptrs], 8]\n" + "fmla v23.4s, v20.4s, v7.4s\n" + "fmla v14.4s, v20.4s, v8.4s\n" + "fmla v16.4s, v25.4s, v12.4s\n" + "ldr q25, [x19, x27]\n" + "fmla v17.4s, v20.4s, v10.4s\n" + "ldr x19, [%[inptrs], 128]\n" + "fmla v2.4s, v20.4s, v11.4s\n" + "fmla v3.4s, v20.4s, v9.4s\n" + "fmla v14.4s, v26.4s, v10.4s\n" + "fmla v0.4s, v27.4s, v6.4s\n" + "mov v15.16b, v13.16b\n" + "fmla v19.4s, v30.4s, v5.4s\n" + "fmla v1.4s, v30.4s, v6.4s\n" + "fmla v16.4s, v30.4s, v9.4s\n" + "fmla v3.4s, v26.4s, v11.4s\n" + "ldr q29, [x18, x27]\n" + "fmla v15.4s, v21.4s, v12.4s\n" + "ldr q27, [x17, x27]\n" + "fmla v0.4s, v30.4s, v8.4s\n" + "ldr q28, [x22, x27]\n" + "fmla v22.4s, v24.4s, v4.4s\n" + "ldr x18, [%[inptrs], 88]\n" + "fmla v19.4s, v24.4s, v7.4s\n" + "ldr x22, [%[inptrs], 256]\n" + "fmla v17.4s, v24.4s, v5.4s\n" + "ldr x17, [%[inptrs], 0]\n" + "fmla v0.4s, v24.4s, v10.4s\n" + "fmla v1.4s, v24.4s, v8.4s\n" + "str q22, [x24, x28]\n" + "mov v18.16b, v13.16b\n" + "fmla v2.4s, v24.4s, v6.4s\n" + "ldr x24, [%[outptrs], 40]\n" + "fmla v16.4s, v24.4s, v11.4s\n" + "fmla v15.4s, v24.4s, v9.4s\n" + "fmla v18.4s, v20.4s, v12.4s\n" + "ldr q22, [x21, x27]\n" + "fmla v23.4s, v25.4s, v4.4s\n" + "ldr x21, [%[inptrs], 216]\n" + "fmla v17.4s, v25.4s, v7.4s\n" + "fmla v14.4s, v25.4s, v5.4s\n" + "fmla v1.4s, v25.4s, v10.4s\n" + "fmla v2.4s, v25.4s, v8.4s\n" + "fmla v3.4s, v25.4s, v6.4s\n" + "fmla v15.4s, v25.4s, v11.4s\n" + "str q23, [x23, x28]\n" + "mov v21.16b, v13.16b\n" + "fmla v18.4s, v25.4s, v9.4s\n" + "ldr x23, [%[outptrs], 16]\n" + "fmla v14.4s, v29.4s, v7.4s\n" + "fmla v2.4s, v29.4s, v10.4s\n" + "fmla v21.4s, v24.4s, v12.4s\n" + "ldr q30, [x20, x27]\n" + "fmla v3.4s, v29.4s, v8.4s\n" + "ldr x20, [%[inptrs], 176]\n" + "fmla v18.4s, v29.4s, v11.4s\n" + "ldr q31, [x19, x27]\n" + "fmla v0.4s, v28.4s, v5.4s\n" + "ldr x19, [%[inptrs], 136]\n" + "fmla v16.4s, v28.4s, v6.4s\n" + "ldr q26, [x18, x27]\n" + "fmla v3.4s, v27.4s, v10.4s\n" + "ldr q23, [x22, x27]\n" + "fmla v19.4s, v22.4s, v4.4s\n" + "ldr x22, [%[inptrs], 264]\n" + "fmla v0.4s, v22.4s, v7.4s\n" + "ldr x18, [%[inptrs], 48]\n" + "fmla v1.4s, v22.4s, v5.4s\n" + "fmla v16.4s, v22.4s, v8.4s\n" + "fmla v15.4s, v22.4s, v6.4s\n" + "fmla v21.4s, v22.4s, v9.4s\n" + "str q19, [x25, x28]\n" + "mov v24.16b, v13.16b\n" + "mov v20.16b, v13.16b\n" + "ldr q27, [x21, x27]\n" + "fmla v17.4s, v30.4s, v4.4s\n" + "ldr x21, [%[inptrs], 224]\n" + "fmla v24.4s, v25.4s, v12.4s\n" + "ldr q28, [x20, x27]\n" + "fmla v1.4s, v30.4s, v7.4s\n" + "ldr x20, [%[inptrs], 184]\n" + "fmla v2.4s, v30.4s, v5.4s\n" + "ldr x25, [%[outptrs], 72]\n" + "str q17, [x24, x28]\n" + "fmla v16.4s, v30.4s, v10.4s\n" + "fmla v15.4s, v30.4s, v8.4s\n" + "ldr q22, [x19, x27]\n" + "fmla v18.4s, v30.4s, v6.4s\n" + "ldr x24, [%[outptrs], 48]\n" + "fmla v21.4s, v30.4s, v11.4s\n" + "ldr x19, [%[inptrs], 96]\n" + "fmla v24.4s, v30.4s, v9.4s\n" + "fmla v20.4s, v30.4s, v12.4s\n" + "fmla v14.4s, v31.4s, v4.4s\n" + "ldr q30, [x22, x27]\n" + "fmla v2.4s, v31.4s, v7.4s\n" + "ldr q19, [x21, x27]\n" + "fmla v3.4s, v31.4s, v5.4s\n" + "ldr x22, [%[inptrs], 272]\n" + "fmla v15.4s, v31.4s, v10.4s\n" + "ldr x21, [%[inptrs], 232]\n" + "str q14, [x23, x28]\n" + "fmla v18.4s, v31.4s, v8.4s\n" + "fmla v24.4s, v31.4s, v11.4s\n" + "ldr q31, [x20, x27]\n" + "fmla v3.4s, v26.4s, v7.4s\n" + "ldr q17, [x22, x27]\n" + "fmla v0.4s, v23.4s, v4.4s\n" + "ldr x22, [%[inptrs], 280]\n" + "fmla v18.4s, v26.4s, v10.4s\n" + "ldr q14, [x21, x27]\n" + "fmla v16.4s, v23.4s, v5.4s\n" + "ldr x23, [%[outptrs], 24]\n" + "fmla v21.4s, v23.4s, v6.4s\n" + "ldr q26, [x22, x27]\n" + "str q0, [x26, x28]\n" + "fmla v1.4s, v27.4s, v4.4s\n" + "fmla v15.4s, v27.4s, v5.4s\n" + "ldr q13, [%[wbptr]]\n" + "fmla v16.4s, v27.4s, v7.4s\n" + "ldr x26, [%[outptrs], 104]\n" + "fmla v21.4s, v27.4s, v8.4s\n" + "add x27, x27, #16\n" + "str q1, [x25, x28]\n" + "fmla v24.4s, v27.4s, v6.4s\n" + "fmla v20.4s, v27.4s, v9.4s\n" + "ldr q12, [%[wbptr], #16]\n" + "fmla v2.4s, v28.4s, v4.4s\n" + "ldr q29, [x17, x27]\n" + "fmla v15.4s, v28.4s, v7.4s\n" + "ldr q27, [x18, x27]\n" + "fmla v18.4s, v28.4s, v5.4s\n" + "ldr x25, [%[outptrs], 80]\n" + "fmla v21.4s, v28.4s, v10.4s\n" + "ldr x17, [%[inptrs], 8]\n" + "str q2, [x24, x28]\n" + "fmla v24.4s, v28.4s, v8.4s\n" + "fmla v20.4s, v28.4s, v11.4s\n" + "ldr q9, [%[wbptr], #64]\n" + "fmla v3.4s, v22.4s, v4.4s\n" + "ldr q28, [x17, x27]\n" + "fmla v18.4s, v22.4s, v7.4s\n" + "ldr q25, [x19, x27]\n" + "fmla v24.4s, v22.4s, v10.4s\n" + "ldr x24, [%[outptrs], 56]\n" + "fmla v16.4s, v30.4s, v4.4s\n" + "ldr q11, [%[wbptr], #32]\n" + "str q3, [x23, x28]\n" + "fmla v21.4s, v30.4s, v5.4s\n" + "fmla v20.4s, v30.4s, v6.4s\n" + "ldr x18, [%[inptrs], 56]\n" + "fmla v15.4s, v19.4s, v4.4s\n" + "ldr x17, [%[inptrs], 16]\n" + "str q16, [x26, x28]\n" + "fmla v24.4s, v19.4s, v5.4s\n" + "fmla v21.4s, v19.4s, v7.4s\n" + "ldr q16, [x18, x27]\n" + "fmla v20.4s, v19.4s, v8.4s\n" + "ldr q6, [%[wbptr], #112]\n" + "str q15, [x25, x28]\n" + "fmla v18.4s, v31.4s, v4.4s\n" + "fmla v24.4s, v31.4s, v7.4s\n" + "ldr q15, [x17, x27]\n" + "fmla v21.4s, v17.4s, v4.4s\n" + "ldr x25, [%[outptrs], 88]\n" + "fmla v20.4s, v31.4s, v10.4s\n" + "ldr q8, [%[wbptr], #80]\n" + "str q18, [x24, x28]\n" + "mov v18.16b, v13.16b\n" + "fmla v24.4s, v14.4s, v4.4s\n" + "ldr x26, [%[outptrs], 112]\n" + "mov v22.16b, v13.16b\n" + "ldr x20, [%[inptrs], 144]\n" + "str q21, [x26, x28]\n" + "fmla v20.4s, v17.4s, v5.4s\n" + "mov v23.16b, v13.16b\n" + "ldr q10, [%[wbptr], #48]\n" + "str q24, [x25, x28]\n" + "mov v19.16b, v13.16b\n" + "mov v17.16b, v13.16b\n" + "ldr q21, [x20, x27]\n" + "fmla v20.4s, v14.4s, v7.4s\n" + "ldr q5, [%[wbptr], #128]\n" + "mov v14.16b, v13.16b\n" + "ldr x26, [%[outptrs], 120]\n" + "mov v0.16b, v13.16b\n" + "ldr x19, [%[inptrs], 104]\n" + "mov v1.16b, v13.16b\n" + "mov v2.16b, v13.16b\n" + "fmla v20.4s, v26.4s, v4.4s\n" + "ldr q7, [%[wbptr], #96]\n" + "fmla v18.4s, v29.4s, v12.4s\n" + "ldr q29, [x19, x27]\n" + "str q20, [x26, x28]\n" + "ldr q4, [%[wbptr], #144]\n" + "add x28, x28, #16\n" + "bne 2b\n" + "3:\n" + "mov v3.16b, v13.16b\n" + "ldr x18, [%[inptrs], 64]\n" + "fmla v18.4s, v27.4s, v9.4s\n" + "ldr x17, [%[inptrs], 24]\n" + "fmla v22.4s, v27.4s, v12.4s\n" + "ldr q30, [x18, x27]\n" + "fmla v23.4s, v28.4s, v12.4s\n" + "ldr x21, [%[inptrs], 192]\n" + "fmla v19.4s, v25.4s, v12.4s\n" + "ldr x20, [%[inptrs], 152]\n" + "fmla v18.4s, v28.4s, v11.4s\n" + "ldr q24, [x17, x27]\n" + "fmla v22.4s, v25.4s, v9.4s\n" + "ldr x19, [%[inptrs], 112]\n" + "fmla v23.4s, v16.4s, v9.4s\n" + "ldr x18, [%[inptrs], 72]\n" + "fmla v17.4s, v16.4s, v12.4s\n" + "ldr x17, [%[inptrs], 32]\n" + "fmla v18.4s, v25.4s, v6.4s\n" + "ldr q31, [x21, x27]\n" + "fmla v22.4s, v16.4s, v11.4s\n" + "ldr x22, [%[inptrs], 240]\n" + "fmla v23.4s, v15.4s, v11.4s\n" + "ldr x21, [%[inptrs], 200]\n" + "fmla v14.4s, v15.4s, v12.4s\n" + "ldr x23, [%[outptrs], 0]\n" + "fmla v18.4s, v16.4s, v8.4s\n" + "ldr q25, [x20, x27]\n" + "fmla v22.4s, v21.4s, v6.4s\n" + "ldr x20, [%[inptrs], 160]\n" + "fmla v19.4s, v21.4s, v9.4s\n" + "ldr x24, [%[outptrs], 32]\n" + "fmla v0.4s, v21.4s, v12.4s\n" + "ldr q21, [x19, x27]\n" + "fmla v18.4s, v15.4s, v10.4s\n" + "ldr q20, [x18, x27]\n" + "fmla v22.4s, v29.4s, v8.4s\n" + "ldr x19, [%[inptrs], 120]\n" + "fmla v23.4s, v29.4s, v6.4s\n" + "ldr x18, [%[inptrs], 80]\n" + "fmla v19.4s, v29.4s, v11.4s\n" + "ldr x25, [%[outptrs], 64]\n" + "fmla v18.4s, v29.4s, v5.4s\n" + "ldr x26, [%[outptrs], 96]\n" + "fmla v17.4s, v29.4s, v9.4s\n" + "add %[wbptr], %[wbptr], #160\n" + "fmla v1.4s, v29.4s, v12.4s\n" + "ldr q26, [x17, x27]\n" + "fmla v22.4s, v30.4s, v10.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v18.4s, v30.4s, v7.4s\n" + "ldr x17, [%[inptrs], 40]\n" + "fmla v23.4s, v30.4s, v8.4s\n" + "fmla v17.4s, v30.4s, v11.4s\n" + "fmla v14.4s, v30.4s, v9.4s\n" + "fmla v2.4s, v30.4s, v12.4s\n" + "mov v16.16b, v13.16b\n" + "fmla v3.4s, v24.4s, v12.4s\n" + "fmla v19.4s, v31.4s, v6.4s\n" + "fmla v0.4s, v31.4s, v9.4s\n" + "mov v15.16b, v13.16b\n" + "fmla v23.4s, v24.4s, v10.4s\n" + "fmla v14.4s, v24.4s, v11.4s\n" + "ldr q27, [x22, x27]\n" + "fmla v22.4s, v25.4s, v5.4s\n" + "ldr x22, [%[inptrs], 248]\n" + "fmla v19.4s, v25.4s, v8.4s\n" + "fmla v17.4s, v25.4s, v6.4s\n" + "fmla v0.4s, v25.4s, v11.4s\n" + "fmla v1.4s, v25.4s, v9.4s\n" + "fmla v16.4s, v25.4s, v12.4s\n" + "ldr q30, [x21, x27]\n" + "fmla v18.4s, v21.4s, v4.4s\n" + "ldr x21, [%[inptrs], 208]\n" + "fmla v22.4s, v21.4s, v7.4s\n" + "fmla v23.4s, v21.4s, v5.4s\n" + "fmla v19.4s, v21.4s, v10.4s\n" + "fmla v17.4s, v21.4s, v8.4s\n" + "fmla v14.4s, v21.4s, v6.4s\n" + "fmla v1.4s, v21.4s, v11.4s\n" + "str q18, [x23, x28]\n" + "mov v18.16b, v13.16b\n" + "fmla v2.4s, v21.4s, v9.4s\n" + "ldr x23, [%[outptrs], 8]\n" + "fmla v15.4s, v21.4s, v12.4s\n" + "ldr q24, [x20, x27]\n" + "fmla v23.4s, v20.4s, v7.4s\n" + "ldr x20, [%[inptrs], 168]\n" + "fmla v17.4s, v20.4s, v10.4s\n" + "fmla v14.4s, v20.4s, v8.4s\n" + "fmla v2.4s, v20.4s, v11.4s\n" + "fmla v3.4s, v20.4s, v9.4s\n" + "fmla v18.4s, v20.4s, v12.4s\n" + "ldr q25, [x19, x27]\n" + "fmla v0.4s, v27.4s, v6.4s\n" + "ldr q29, [x18, x27]\n" + "fmla v14.4s, v26.4s, v10.4s\n" + "ldr x19, [%[inptrs], 128]\n" + "fmla v3.4s, v26.4s, v11.4s\n" + "ldr q27, [x17, x27]\n" + "fmla v19.4s, v30.4s, v5.4s\n" + "ldr x18, [%[inptrs], 88]\n" + "fmla v0.4s, v30.4s, v8.4s\n" + "fmla v1.4s, v30.4s, v6.4s\n" + "fmla v16.4s, v30.4s, v9.4s\n" + "ldr q28, [x22, x27]\n" + "fmla v22.4s, v24.4s, v4.4s\n" + "ldr x22, [%[inptrs], 256]\n" + "fmla v19.4s, v24.4s, v7.4s\n" + "fmla v17.4s, v24.4s, v5.4s\n" + "fmla v0.4s, v24.4s, v10.4s\n" + "fmla v1.4s, v24.4s, v8.4s\n" + "fmla v2.4s, v24.4s, v6.4s\n" + "fmla v16.4s, v24.4s, v11.4s\n" + "str q22, [x24, x28]\n" + "mov v21.16b, v13.16b\n" + "fmla v15.4s, v24.4s, v9.4s\n" + "ldr x24, [%[outptrs], 40]\n" + "fmla v23.4s, v25.4s, v4.4s\n" + "fmla v17.4s, v25.4s, v7.4s\n" + "fmla v21.4s, v24.4s, v12.4s\n" + "ldr q22, [x21, x27]\n" + "fmla v14.4s, v25.4s, v5.4s\n" + "ldr x21, [%[inptrs], 216]\n" + "fmla v1.4s, v25.4s, v10.4s\n" + "fmla v2.4s, v25.4s, v8.4s\n" + "str q23, [x23, x28]\n" + "mov v24.16b, v13.16b\n" + "mov v20.16b, v13.16b\n" + "ldr x23, [%[outptrs], 16]\n" + "fmla v3.4s, v25.4s, v6.4s\n" + "fmla v15.4s, v25.4s, v11.4s\n" + "fmla v18.4s, v25.4s, v9.4s\n" + "fmla v24.4s, v25.4s, v12.4s\n" + "fmla v14.4s, v29.4s, v7.4s\n" + "ldr q30, [x20, x27]\n" + "fmla v2.4s, v29.4s, v10.4s\n" + "ldr x20, [%[inptrs], 176]\n" + "fmla v3.4s, v29.4s, v8.4s\n" + "fmla v0.4s, v28.4s, v5.4s\n" + "fmla v18.4s, v29.4s, v11.4s\n" + "ldr q31, [x19, x27]\n" + "fmla v16.4s, v28.4s, v6.4s\n" + "ldr q26, [x18, x27]\n" + "fmla v19.4s, v22.4s, v4.4s\n" + "ldr x19, [%[inptrs], 136]\n" + "fmla v3.4s, v27.4s, v10.4s\n" + "ldr q23, [x22, x27]\n" + "fmla v0.4s, v22.4s, v7.4s\n" + "ldr x22, [%[inptrs], 264]\n" + "fmla v1.4s, v22.4s, v5.4s\n" + "fmla v16.4s, v22.4s, v8.4s\n" + "str q19, [x25, x28]\n" + "fmla v15.4s, v22.4s, v6.4s\n" + "fmla v21.4s, v22.4s, v9.4s\n" + "ldr q27, [x21, x27]\n" + "fmla v17.4s, v30.4s, v4.4s\n" + "ldr q28, [x20, x27]\n" + "fmla v1.4s, v30.4s, v7.4s\n" + "ldr x21, [%[inptrs], 224]\n" + "fmla v2.4s, v30.4s, v5.4s\n" + "ldr x20, [%[inptrs], 184]\n" + "fmla v16.4s, v30.4s, v10.4s\n" + "ldr x25, [%[outptrs], 72]\n" + "str q17, [x24, x28]\n" + "fmla v15.4s, v30.4s, v8.4s\n" + "fmla v18.4s, v30.4s, v6.4s\n" + "ldr q22, [x19, x27]\n" + "fmla v21.4s, v30.4s, v11.4s\n" + "ldr x24, [%[outptrs], 48]\n" + "fmla v24.4s, v30.4s, v9.4s\n" + "fmla v20.4s, v30.4s, v12.4s\n" + "fmla v14.4s, v31.4s, v4.4s\n" + "ldr q30, [x22, x27]\n" + "fmla v2.4s, v31.4s, v7.4s\n" + "ldr q19, [x21, x27]\n" + "fmla v3.4s, v31.4s, v5.4s\n" + "ldr x22, [%[inptrs], 272]\n" + "fmla v15.4s, v31.4s, v10.4s\n" + "ldr x21, [%[inptrs], 232]\n" + "str q14, [x23, x28]\n" + "fmla v18.4s, v31.4s, v8.4s\n" + "fmla v24.4s, v31.4s, v11.4s\n" + "ldr q31, [x20, x27]\n" + "fmla v3.4s, v26.4s, v7.4s\n" + "ldr q17, [x22, x27]\n" + "fmla v0.4s, v23.4s, v4.4s\n" + "ldr x22, [%[inptrs], 280]\n" + "fmla v18.4s, v26.4s, v10.4s\n" + "ldr q14, [x21, x27]\n" + "fmla v16.4s, v23.4s, v5.4s\n" + "ldr x23, [%[outptrs], 24]\n" + "fmla v21.4s, v23.4s, v6.4s\n" + "ldr q26, [x22, x27]\n" + "str q0, [x26, x28]\n" + "fmla v1.4s, v27.4s, v4.4s\n" + "fmla v15.4s, v27.4s, v5.4s\n" + "ldr x26, [%[outptrs], 104]\n" + "fmla v16.4s, v27.4s, v7.4s\n" + "add x27, x27, #16\n" + "fmla v21.4s, v27.4s, v8.4s\n" + "fmla v24.4s, v27.4s, v6.4s\n" + "str q1, [x25, x28]\n" + "fmla v20.4s, v27.4s, v9.4s\n" + "fmla v2.4s, v28.4s, v4.4s\n" + "ldr x25, [%[outptrs], 80]\n" + "fmla v15.4s, v28.4s, v7.4s\n" + "fmla v18.4s, v28.4s, v5.4s\n" + "fmla v21.4s, v28.4s, v10.4s\n" + "fmla v24.4s, v28.4s, v8.4s\n" + "fmla v20.4s, v28.4s, v11.4s\n" + "fmla v3.4s, v22.4s, v4.4s\n" + "str q2, [x24, x28]\n" + "fmla v16.4s, v30.4s, v4.4s\n" + "fmla v18.4s, v22.4s, v7.4s\n" + "ldr x24, [%[outptrs], 56]\n" + "fmla v24.4s, v22.4s, v10.4s\n" + "fmla v21.4s, v30.4s, v5.4s\n" + "str q3, [x23, x28]\n" + "fmla v20.4s, v30.4s, v6.4s\n" + "str q16, [x26, x28]\n" + "fmla v15.4s, v19.4s, v4.4s\n" + "fmla v18.4s, v31.4s, v4.4s\n" + "ldr x26, [%[outptrs], 112]\n" + "fmla v21.4s, v19.4s, v7.4s\n" + "fmla v24.4s, v19.4s, v5.4s\n" + "fmla v20.4s, v19.4s, v8.4s\n" + "str q15, [x25, x28]\n" + "str q18, [x24, x28]\n" + "ldr x25, [%[outptrs], 88]\n" + "fmla v24.4s, v31.4s, v7.4s\n" + "fmla v21.4s, v17.4s, v4.4s\n" + "fmla v20.4s, v31.4s, v10.4s\n" + "str q21, [x26, x28]\n" + "fmla v20.4s, v17.4s, v5.4s\n" + "ldr x26, [%[outptrs], 120]\n" + "fmla v24.4s, v14.4s, v4.4s\n" + "fmla v20.4s, v14.4s, v7.4s\n" + "str q24, [x25, x28]\n" + "fmla v20.4s, v26.4s, v4.4s\n" + "str q20, [x26, x28]\n" + "add x28, x28, #16\n" + "4:\n" + "cbz x15, 7f\n" + "ldr s13, [%[wbptr]]\n" + "mov v18.16b, v13.16b\n" + "ldr s12, [%[wbptr], #4]\n" + "mov v22.16b, v13.16b\n" + "ldr s11, [%[wbptr], #8]\n" + "mov v23.16b, v13.16b\n" + "ldr s10, [%[wbptr], #12]\n" + "mov v19.16b, v13.16b\n" + "ldr s9, [%[wbptr], #16]\n" + "mov v17.16b, v13.16b\n" + "ldr s8, [%[wbptr], #20]\n" + "mov v14.16b, v13.16b\n" + "ldr s7, [%[wbptr], #24]\n" + "mov v0.16b, v13.16b\n" + "ldr s6, [%[wbptr], #28]\n" + "mov v1.16b, v13.16b\n" + "ldr s5, [%[wbptr], #32]\n" + "mov v2.16b, v13.16b\n" + "ldr s4, [%[wbptr], #36]\n" + "ldr x17, [%[inptrs], 0]\n" + "ldr x18, [%[inptrs], 48]\n" + "ldr x19, [%[inptrs], 96]\n" + "ldr x20, [%[inptrs], 144]\n" + "subs x15, x15, #1\n" + "ldr s29, [x17, x27]\n" + "fmla v18.4s, v29.4s, v12.4s\n" + "ldr s27, [x18, x27]\n" + "ldr s25, [x19, x27]\n" + "ldr x17, [%[inptrs], 8]\n" + "ldr s21, [x20, x27]\n" + "ldr x18, [%[inptrs], 56]\n" + "ldr s28, [x17, x27]\n" + "ldr x19, [%[inptrs], 104]\n" + "ldr s16, [x18, x27]\n" + "ldr x17, [%[inptrs], 16]\n" + "ldr s29, [x19, x27]\n" + "ldr s15, [x17, x27]\n" + "beq 6f\n" + "5:\n" + "mov v3.16b, v13.16b\n" + "ldr x18, [%[inptrs], 64]\n" + "fmla v18.4s, v27.4s, v9.4s\n" + "ldr x17, [%[inptrs], 24]\n" + "fmla v22.4s, v27.4s, v12.4s\n" + "ldr s30, [x18, x27]\n" + "fmla v23.4s, v28.4s, v12.4s\n" + "ldr x21, [%[inptrs], 192]\n" + "fmla v19.4s, v25.4s, v12.4s\n" + "ldr x20, [%[inptrs], 152]\n" + "fmla v18.4s, v28.4s, v11.4s\n" + "ldr s24, [x17, x27]\n" + "fmla v22.4s, v25.4s, v9.4s\n" + "ldr x19, [%[inptrs], 112]\n" + "fmla v23.4s, v16.4s, v9.4s\n" + "ldr x18, [%[inptrs], 72]\n" + "fmla v17.4s, v16.4s, v12.4s\n" + "ldr x17, [%[inptrs], 32]\n" + "fmla v18.4s, v25.4s, v6.4s\n" + "ldr s31, [x21, x27]\n" + "fmla v22.4s, v16.4s, v11.4s\n" + "ldr x22, [%[inptrs], 240]\n" + "fmla v23.4s, v15.4s, v11.4s\n" + "ldr x21, [%[inptrs], 200]\n" + "fmla v14.4s, v15.4s, v12.4s\n" + "ldr x23, [%[outptrs], 0]\n" + "fmla v18.4s, v16.4s, v8.4s\n" + "ldr s25, [x20, x27]\n" + "fmla v22.4s, v21.4s, v6.4s\n" + "ldr x20, [%[inptrs], 160]\n" + "fmla v19.4s, v21.4s, v9.4s\n" + "ldr x24, [%[outptrs], 32]\n" + "fmla v0.4s, v21.4s, v12.4s\n" + "ldr s21, [x19, x27]\n" + "fmla v18.4s, v15.4s, v10.4s\n" + "ldr s20, [x18, x27]\n" + "fmla v22.4s, v29.4s, v8.4s\n" + "ldr x19, [%[inptrs], 120]\n" + "fmla v23.4s, v29.4s, v6.4s\n" + "ldr x18, [%[inptrs], 80]\n" + "fmla v19.4s, v29.4s, v11.4s\n" + "ldr x25, [%[outptrs], 64]\n" + "fmla v18.4s, v29.4s, v5.4s\n" + "ldr x26, [%[outptrs], 96]\n" + "fmla v17.4s, v29.4s, v9.4s\n" + "add %[wbptr], %[wbptr], #40\n" + "fmla v1.4s, v29.4s, v12.4s\n" + "ldr s26, [x17, x27]\n" + "fmla v22.4s, v30.4s, v10.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v18.4s, v30.4s, v7.4s\n" + "ldr x17, [%[inptrs], 40]\n" + "fmla v23.4s, v30.4s, v8.4s\n" + "subs x15, x15, #1\n" + "fmla v17.4s, v30.4s, v11.4s\n" + "fmla v14.4s, v30.4s, v9.4s\n" + "fmla v2.4s, v30.4s, v12.4s\n" + "ldr s27, [x22, x27]\n" + "fmla v3.4s, v24.4s, v12.4s\n" + "ldr x22, [%[inptrs], 248]\n" + "fmla v23.4s, v24.4s, v10.4s\n" + "fmla v19.4s, v31.4s, v6.4s\n" + "fmla v14.4s, v24.4s, v11.4s\n" + "ldr s30, [x21, x27]\n" + "fmla v0.4s, v31.4s, v9.4s\n" + "ldr s24, [x20, x27]\n" + "fmla v22.4s, v25.4s, v5.4s\n" + "ldr x21, [%[inptrs], 208]\n" + "fmla v19.4s, v25.4s, v8.4s\n" + "ldr x20, [%[inptrs], 168]\n" + "fmla v17.4s, v25.4s, v6.4s\n" + "fmla v1.4s, v25.4s, v9.4s\n" + "fmla v0.4s, v25.4s, v11.4s\n" + "fmla v18.4s, v21.4s, v4.4s\n" + "fmla v22.4s, v21.4s, v7.4s\n" + "fmla v23.4s, v21.4s, v5.4s\n" + "fmla v19.4s, v21.4s, v10.4s\n" + "fmla v14.4s, v21.4s, v6.4s\n" + "fmla v17.4s, v21.4s, v8.4s\n" + "fmla v1.4s, v21.4s, v11.4s\n" + "str s18, [x23, x28]\n" + "mov v16.16b, v13.16b\n" + "fmla v2.4s, v21.4s, v9.4s\n" + "ldr x23, [%[outptrs], 8]\n" + "fmla v23.4s, v20.4s, v7.4s\n" + "fmla v14.4s, v20.4s, v8.4s\n" + "fmla v16.4s, v25.4s, v12.4s\n" + "ldr s25, [x19, x27]\n" + "fmla v17.4s, v20.4s, v10.4s\n" + "ldr x19, [%[inptrs], 128]\n" + "fmla v2.4s, v20.4s, v11.4s\n" + "fmla v3.4s, v20.4s, v9.4s\n" + "fmla v14.4s, v26.4s, v10.4s\n" + "fmla v0.4s, v27.4s, v6.4s\n" + "mov v15.16b, v13.16b\n" + "fmla v19.4s, v30.4s, v5.4s\n" + "fmla v1.4s, v30.4s, v6.4s\n" + "fmla v16.4s, v30.4s, v9.4s\n" + "fmla v3.4s, v26.4s, v11.4s\n" + "ldr s29, [x18, x27]\n" + "fmla v15.4s, v21.4s, v12.4s\n" + "ldr s27, [x17, x27]\n" + "fmla v0.4s, v30.4s, v8.4s\n" + "ldr s28, [x22, x27]\n" + "fmla v22.4s, v24.4s, v4.4s\n" + "ldr x18, [%[inptrs], 88]\n" + "fmla v19.4s, v24.4s, v7.4s\n" + "ldr x22, [%[inptrs], 256]\n" + "fmla v17.4s, v24.4s, v5.4s\n" + "ldr x17, [%[inptrs], 0]\n" + "fmla v0.4s, v24.4s, v10.4s\n" + "fmla v1.4s, v24.4s, v8.4s\n" + "str s22, [x24, x28]\n" + "mov v18.16b, v13.16b\n" + "fmla v2.4s, v24.4s, v6.4s\n" + "ldr x24, [%[outptrs], 40]\n" + "fmla v16.4s, v24.4s, v11.4s\n" + "fmla v15.4s, v24.4s, v9.4s\n" + "fmla v18.4s, v20.4s, v12.4s\n" + "ldr s22, [x21, x27]\n" + "fmla v23.4s, v25.4s, v4.4s\n" + "ldr x21, [%[inptrs], 216]\n" + "fmla v17.4s, v25.4s, v7.4s\n" + "fmla v14.4s, v25.4s, v5.4s\n" + "fmla v1.4s, v25.4s, v10.4s\n" + "fmla v2.4s, v25.4s, v8.4s\n" + "fmla v3.4s, v25.4s, v6.4s\n" + "fmla v15.4s, v25.4s, v11.4s\n" + "str s23, [x23, x28]\n" + "mov v21.16b, v13.16b\n" + "fmla v18.4s, v25.4s, v9.4s\n" + "ldr x23, [%[outptrs], 16]\n" + "fmla v14.4s, v29.4s, v7.4s\n" + "fmla v2.4s, v29.4s, v10.4s\n" + "fmla v21.4s, v24.4s, v12.4s\n" + "ldr s30, [x20, x27]\n" + "fmla v3.4s, v29.4s, v8.4s\n" + "ldr x20, [%[inptrs], 176]\n" + "fmla v18.4s, v29.4s, v11.4s\n" + "ldr s31, [x19, x27]\n" + "fmla v0.4s, v28.4s, v5.4s\n" + "ldr x19, [%[inptrs], 136]\n" + "fmla v16.4s, v28.4s, v6.4s\n" + "ldr s26, [x18, x27]\n" + "fmla v3.4s, v27.4s, v10.4s\n" + "ldr s23, [x22, x27]\n" + "fmla v19.4s, v22.4s, v4.4s\n" + "ldr x22, [%[inptrs], 264]\n" + "fmla v0.4s, v22.4s, v7.4s\n" + "ldr x18, [%[inptrs], 48]\n" + "fmla v1.4s, v22.4s, v5.4s\n" + "fmla v16.4s, v22.4s, v8.4s\n" + "fmla v15.4s, v22.4s, v6.4s\n" + "fmla v21.4s, v22.4s, v9.4s\n" + "str s19, [x25, x28]\n" + "mov v24.16b, v13.16b\n" + "mov v20.16b, v13.16b\n" + "ldr s27, [x21, x27]\n" + "fmla v17.4s, v30.4s, v4.4s\n" + "ldr x21, [%[inptrs], 224]\n" + "fmla v24.4s, v25.4s, v12.4s\n" + "ldr s28, [x20, x27]\n" + "fmla v1.4s, v30.4s, v7.4s\n" + "ldr x20, [%[inptrs], 184]\n" + "fmla v2.4s, v30.4s, v5.4s\n" + "ldr x25, [%[outptrs], 72]\n" + "str s17, [x24, x28]\n" + "fmla v16.4s, v30.4s, v10.4s\n" + "fmla v15.4s, v30.4s, v8.4s\n" + "ldr s22, [x19, x27]\n" + "fmla v18.4s, v30.4s, v6.4s\n" + "ldr x24, [%[outptrs], 48]\n" + "fmla v21.4s, v30.4s, v11.4s\n" + "ldr x19, [%[inptrs], 96]\n" + "fmla v24.4s, v30.4s, v9.4s\n" + "fmla v20.4s, v30.4s, v12.4s\n" + "fmla v14.4s, v31.4s, v4.4s\n" + "ldr s30, [x22, x27]\n" + "fmla v2.4s, v31.4s, v7.4s\n" + "ldr s19, [x21, x27]\n" + "fmla v3.4s, v31.4s, v5.4s\n" + "ldr x22, [%[inptrs], 272]\n" + "fmla v15.4s, v31.4s, v10.4s\n" + "ldr x21, [%[inptrs], 232]\n" + "str s14, [x23, x28]\n" + "fmla v18.4s, v31.4s, v8.4s\n" + "fmla v24.4s, v31.4s, v11.4s\n" + "ldr s31, [x20, x27]\n" + "fmla v3.4s, v26.4s, v7.4s\n" + "ldr s17, [x22, x27]\n" + "fmla v0.4s, v23.4s, v4.4s\n" + "ldr x22, [%[inptrs], 280]\n" + "fmla v18.4s, v26.4s, v10.4s\n" + "ldr s14, [x21, x27]\n" + "fmla v16.4s, v23.4s, v5.4s\n" + "ldr x23, [%[outptrs], 24]\n" + "fmla v21.4s, v23.4s, v6.4s\n" + "ldr s26, [x22, x27]\n" + "str s0, [x26, x28]\n" + "fmla v1.4s, v27.4s, v4.4s\n" + "fmla v15.4s, v27.4s, v5.4s\n" + "ldr s13, [%[wbptr]]\n" + "fmla v16.4s, v27.4s, v7.4s\n" + "ldr x26, [%[outptrs], 104]\n" + "fmla v21.4s, v27.4s, v8.4s\n" + "add x27, x27, #4\n" + "str s1, [x25, x28]\n" + "fmla v24.4s, v27.4s, v6.4s\n" + "fmla v20.4s, v27.4s, v9.4s\n" + "ldr s12, [%[wbptr], #4]\n" + "fmla v2.4s, v28.4s, v4.4s\n" + "ldr s29, [x17, x27]\n" + "fmla v15.4s, v28.4s, v7.4s\n" + "ldr s27, [x18, x27]\n" + "fmla v18.4s, v28.4s, v5.4s\n" + "ldr x25, [%[outptrs], 80]\n" + "fmla v21.4s, v28.4s, v10.4s\n" + "ldr x17, [%[inptrs], 8]\n" + "str s2, [x24, x28]\n" + "fmla v24.4s, v28.4s, v8.4s\n" + "fmla v20.4s, v28.4s, v11.4s\n" + "ldr s9, [%[wbptr], #16]\n" + "fmla v3.4s, v22.4s, v4.4s\n" + "ldr s28, [x17, x27]\n" + "fmla v18.4s, v22.4s, v7.4s\n" + "ldr s25, [x19, x27]\n" + "fmla v24.4s, v22.4s, v10.4s\n" + "ldr x24, [%[outptrs], 56]\n" + "fmla v16.4s, v30.4s, v4.4s\n" + "ldr s11, [%[wbptr], #8]\n" + "str s3, [x23, x28]\n" + "fmla v21.4s, v30.4s, v5.4s\n" + "fmla v20.4s, v30.4s, v6.4s\n" + "ldr x18, [%[inptrs], 56]\n" + "fmla v15.4s, v19.4s, v4.4s\n" + "ldr x17, [%[inptrs], 16]\n" + "str s16, [x26, x28]\n" + "fmla v24.4s, v19.4s, v5.4s\n" + "fmla v21.4s, v19.4s, v7.4s\n" + "ldr s16, [x18, x27]\n" + "fmla v20.4s, v19.4s, v8.4s\n" + "ldr s6, [%[wbptr], #28]\n" + "str s15, [x25, x28]\n" + "fmla v18.4s, v31.4s, v4.4s\n" + "fmla v24.4s, v31.4s, v7.4s\n" + "ldr s15, [x17, x27]\n" + "fmla v21.4s, v17.4s, v4.4s\n" + "ldr x25, [%[outptrs], 88]\n" + "fmla v20.4s, v31.4s, v10.4s\n" + "ldr s8, [%[wbptr], #20]\n" + "str s18, [x24, x28]\n" + "mov v18.16b, v13.16b\n" + "fmla v24.4s, v14.4s, v4.4s\n" + "ldr x26, [%[outptrs], 112]\n" + "mov v22.16b, v13.16b\n" + "ldr x20, [%[inptrs], 144]\n" + "str s21, [x26, x28]\n" + "fmla v20.4s, v17.4s, v5.4s\n" + "mov v23.16b, v13.16b\n" + "ldr s10, [%[wbptr], #12]\n" + "str s24, [x25, x28]\n" + "mov v19.16b, v13.16b\n" + "mov v17.16b, v13.16b\n" + "ldr s21, [x20, x27]\n" + "fmla v20.4s, v14.4s, v7.4s\n" + "ldr s5, [%[wbptr], #32]\n" + "mov v14.16b, v13.16b\n" + "ldr x26, [%[outptrs], 120]\n" + "mov v0.16b, v13.16b\n" + "ldr x19, [%[inptrs], 104]\n" + "mov v1.16b, v13.16b\n" + "mov v2.16b, v13.16b\n" + "fmla v20.4s, v26.4s, v4.4s\n" + "ldr s7, [%[wbptr], #24]\n" + "fmla v18.4s, v29.4s, v12.4s\n" + "ldr s29, [x19, x27]\n" + "str s20, [x26, x28]\n" + "ldr s4, [%[wbptr], #36]\n" + "add x28, x28, #4\n" + "bne 5b\n" + "6:\n" + "mov v3.16b, v13.16b\n" + "ldr x18, [%[inptrs], 64]\n" + "fmla v18.4s, v27.4s, v9.4s\n" + "ldr x17, [%[inptrs], 24]\n" + "fmla v22.4s, v27.4s, v12.4s\n" + "ldr s30, [x18, x27]\n" + "fmla v23.4s, v28.4s, v12.4s\n" + "ldr x21, [%[inptrs], 192]\n" + "fmla v19.4s, v25.4s, v12.4s\n" + "ldr x20, [%[inptrs], 152]\n" + "fmla v18.4s, v28.4s, v11.4s\n" + "ldr s24, [x17, x27]\n" + "fmla v22.4s, v25.4s, v9.4s\n" + "ldr x19, [%[inptrs], 112]\n" + "fmla v23.4s, v16.4s, v9.4s\n" + "ldr x18, [%[inptrs], 72]\n" + "fmla v17.4s, v16.4s, v12.4s\n" + "ldr x17, [%[inptrs], 32]\n" + "fmla v18.4s, v25.4s, v6.4s\n" + "ldr s31, [x21, x27]\n" + "fmla v22.4s, v16.4s, v11.4s\n" + "ldr x22, [%[inptrs], 240]\n" + "fmla v23.4s, v15.4s, v11.4s\n" + "ldr x21, [%[inptrs], 200]\n" + "fmla v14.4s, v15.4s, v12.4s\n" + "ldr x23, [%[outptrs], 0]\n" + "fmla v18.4s, v16.4s, v8.4s\n" + "ldr s25, [x20, x27]\n" + "fmla v22.4s, v21.4s, v6.4s\n" + "ldr x20, [%[inptrs], 160]\n" + "fmla v19.4s, v21.4s, v9.4s\n" + "ldr x24, [%[outptrs], 32]\n" + "fmla v0.4s, v21.4s, v12.4s\n" + "ldr s21, [x19, x27]\n" + "fmla v18.4s, v15.4s, v10.4s\n" + "ldr s20, [x18, x27]\n" + "fmla v22.4s, v29.4s, v8.4s\n" + "ldr x19, [%[inptrs], 120]\n" + "fmla v23.4s, v29.4s, v6.4s\n" + "ldr x18, [%[inptrs], 80]\n" + "fmla v19.4s, v29.4s, v11.4s\n" + "ldr x25, [%[outptrs], 64]\n" + "fmla v18.4s, v29.4s, v5.4s\n" + "ldr x26, [%[outptrs], 96]\n" + "fmla v17.4s, v29.4s, v9.4s\n" + "add %[wbptr], %[wbptr], #40\n" + "fmla v1.4s, v29.4s, v12.4s\n" + "ldr s26, [x17, x27]\n" + "fmla v22.4s, v30.4s, v10.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v18.4s, v30.4s, v7.4s\n" + "ldr x17, [%[inptrs], 40]\n" + "fmla v23.4s, v30.4s, v8.4s\n" + "fmla v17.4s, v30.4s, v11.4s\n" + "fmla v14.4s, v30.4s, v9.4s\n" + "fmla v2.4s, v30.4s, v12.4s\n" + "mov v16.16b, v13.16b\n" + "fmla v3.4s, v24.4s, v12.4s\n" + "fmla v19.4s, v31.4s, v6.4s\n" + "fmla v0.4s, v31.4s, v9.4s\n" + "mov v15.16b, v13.16b\n" + "fmla v23.4s, v24.4s, v10.4s\n" + "fmla v14.4s, v24.4s, v11.4s\n" + "ldr s27, [x22, x27]\n" + "fmla v22.4s, v25.4s, v5.4s\n" + "ldr x22, [%[inptrs], 248]\n" + "fmla v19.4s, v25.4s, v8.4s\n" + "fmla v17.4s, v25.4s, v6.4s\n" + "fmla v0.4s, v25.4s, v11.4s\n" + "fmla v1.4s, v25.4s, v9.4s\n" + "fmla v16.4s, v25.4s, v12.4s\n" + "ldr s30, [x21, x27]\n" + "fmla v18.4s, v21.4s, v4.4s\n" + "ldr x21, [%[inptrs], 208]\n" + "fmla v22.4s, v21.4s, v7.4s\n" + "fmla v23.4s, v21.4s, v5.4s\n" + "fmla v19.4s, v21.4s, v10.4s\n" + "fmla v17.4s, v21.4s, v8.4s\n" + "fmla v14.4s, v21.4s, v6.4s\n" + "fmla v1.4s, v21.4s, v11.4s\n" + "str s18, [x23, x28]\n" + "mov v18.16b, v13.16b\n" + "fmla v2.4s, v21.4s, v9.4s\n" + "ldr x23, [%[outptrs], 8]\n" + "fmla v15.4s, v21.4s, v12.4s\n" + "ldr s24, [x20, x27]\n" + "fmla v23.4s, v20.4s, v7.4s\n" + "ldr x20, [%[inptrs], 168]\n" + "fmla v17.4s, v20.4s, v10.4s\n" + "fmla v14.4s, v20.4s, v8.4s\n" + "fmla v2.4s, v20.4s, v11.4s\n" + "fmla v3.4s, v20.4s, v9.4s\n" + "fmla v18.4s, v20.4s, v12.4s\n" + "ldr s25, [x19, x27]\n" + "fmla v0.4s, v27.4s, v6.4s\n" + "ldr s29, [x18, x27]\n" + "fmla v14.4s, v26.4s, v10.4s\n" + "ldr x19, [%[inptrs], 128]\n" + "fmla v3.4s, v26.4s, v11.4s\n" + "ldr s27, [x17, x27]\n" + "fmla v19.4s, v30.4s, v5.4s\n" + "ldr x18, [%[inptrs], 88]\n" + "fmla v0.4s, v30.4s, v8.4s\n" + "fmla v1.4s, v30.4s, v6.4s\n" + "fmla v16.4s, v30.4s, v9.4s\n" + "ldr s28, [x22, x27]\n" + "fmla v22.4s, v24.4s, v4.4s\n" + "ldr x22, [%[inptrs], 256]\n" + "fmla v19.4s, v24.4s, v7.4s\n" + "fmla v17.4s, v24.4s, v5.4s\n" + "fmla v0.4s, v24.4s, v10.4s\n" + "fmla v1.4s, v24.4s, v8.4s\n" + "fmla v2.4s, v24.4s, v6.4s\n" + "fmla v16.4s, v24.4s, v11.4s\n" + "str s22, [x24, x28]\n" + "mov v21.16b, v13.16b\n" + "fmla v15.4s, v24.4s, v9.4s\n" + "ldr x24, [%[outptrs], 40]\n" + "fmla v23.4s, v25.4s, v4.4s\n" + "fmla v17.4s, v25.4s, v7.4s\n" + "fmla v21.4s, v24.4s, v12.4s\n" + "ldr s22, [x21, x27]\n" + "fmla v14.4s, v25.4s, v5.4s\n" + "ldr x21, [%[inptrs], 216]\n" + "fmla v1.4s, v25.4s, v10.4s\n" + "fmla v2.4s, v25.4s, v8.4s\n" + "str s23, [x23, x28]\n" + "mov v24.16b, v13.16b\n" + "mov v20.16b, v13.16b\n" + "ldr x23, [%[outptrs], 16]\n" + "fmla v3.4s, v25.4s, v6.4s\n" + "fmla v15.4s, v25.4s, v11.4s\n" + "fmla v18.4s, v25.4s, v9.4s\n" + "fmla v24.4s, v25.4s, v12.4s\n" + "fmla v14.4s, v29.4s, v7.4s\n" + "ldr s30, [x20, x27]\n" + "fmla v2.4s, v29.4s, v10.4s\n" + "ldr x20, [%[inptrs], 176]\n" + "fmla v3.4s, v29.4s, v8.4s\n" + "fmla v0.4s, v28.4s, v5.4s\n" + "fmla v18.4s, v29.4s, v11.4s\n" + "ldr s31, [x19, x27]\n" + "fmla v16.4s, v28.4s, v6.4s\n" + "ldr s26, [x18, x27]\n" + "fmla v19.4s, v22.4s, v4.4s\n" + "ldr x19, [%[inptrs], 136]\n" + "fmla v3.4s, v27.4s, v10.4s\n" + "ldr s23, [x22, x27]\n" + "fmla v0.4s, v22.4s, v7.4s\n" + "ldr x22, [%[inptrs], 264]\n" + "fmla v1.4s, v22.4s, v5.4s\n" + "fmla v16.4s, v22.4s, v8.4s\n" + "str s19, [x25, x28]\n" + "fmla v15.4s, v22.4s, v6.4s\n" + "fmla v21.4s, v22.4s, v9.4s\n" + "ldr s27, [x21, x27]\n" + "fmla v17.4s, v30.4s, v4.4s\n" + "ldr s28, [x20, x27]\n" + "fmla v1.4s, v30.4s, v7.4s\n" + "ldr x21, [%[inptrs], 224]\n" + "fmla v2.4s, v30.4s, v5.4s\n" + "ldr x20, [%[inptrs], 184]\n" + "fmla v16.4s, v30.4s, v10.4s\n" + "ldr x25, [%[outptrs], 72]\n" + "str s17, [x24, x28]\n" + "fmla v15.4s, v30.4s, v8.4s\n" + "fmla v18.4s, v30.4s, v6.4s\n" + "ldr s22, [x19, x27]\n" + "fmla v21.4s, v30.4s, v11.4s\n" + "ldr x24, [%[outptrs], 48]\n" + "fmla v24.4s, v30.4s, v9.4s\n" + "fmla v20.4s, v30.4s, v12.4s\n" + "fmla v14.4s, v31.4s, v4.4s\n" + "ldr s30, [x22, x27]\n" + "fmla v2.4s, v31.4s, v7.4s\n" + "ldr s19, [x21, x27]\n" + "fmla v3.4s, v31.4s, v5.4s\n" + "ldr x22, [%[inptrs], 272]\n" + "fmla v15.4s, v31.4s, v10.4s\n" + "ldr x21, [%[inptrs], 232]\n" + "str s14, [x23, x28]\n" + "fmla v18.4s, v31.4s, v8.4s\n" + "fmla v24.4s, v31.4s, v11.4s\n" + "ldr s31, [x20, x27]\n" + "fmla v3.4s, v26.4s, v7.4s\n" + "ldr s17, [x22, x27]\n" + "fmla v0.4s, v23.4s, v4.4s\n" + "ldr x22, [%[inptrs], 280]\n" + "fmla v18.4s, v26.4s, v10.4s\n" + "ldr s14, [x21, x27]\n" + "fmla v16.4s, v23.4s, v5.4s\n" + "ldr x23, [%[outptrs], 24]\n" + "fmla v21.4s, v23.4s, v6.4s\n" + "ldr s26, [x22, x27]\n" + "str s0, [x26, x28]\n" + "fmla v1.4s, v27.4s, v4.4s\n" + "fmla v15.4s, v27.4s, v5.4s\n" + "ldr x26, [%[outptrs], 104]\n" + "fmla v16.4s, v27.4s, v7.4s\n" + "add x27, x27, #4\n" + "fmla v21.4s, v27.4s, v8.4s\n" + "fmla v24.4s, v27.4s, v6.4s\n" + "str s1, [x25, x28]\n" + "fmla v20.4s, v27.4s, v9.4s\n" + "fmla v2.4s, v28.4s, v4.4s\n" + "ldr x25, [%[outptrs], 80]\n" + "fmla v15.4s, v28.4s, v7.4s\n" + "fmla v18.4s, v28.4s, v5.4s\n" + "fmla v21.4s, v28.4s, v10.4s\n" + "fmla v24.4s, v28.4s, v8.4s\n" + "fmla v20.4s, v28.4s, v11.4s\n" + "fmla v3.4s, v22.4s, v4.4s\n" + "str s2, [x24, x28]\n" + "fmla v16.4s, v30.4s, v4.4s\n" + "fmla v18.4s, v22.4s, v7.4s\n" + "ldr x24, [%[outptrs], 56]\n" + "fmla v24.4s, v22.4s, v10.4s\n" + "fmla v21.4s, v30.4s, v5.4s\n" + "str s3, [x23, x28]\n" + "fmla v20.4s, v30.4s, v6.4s\n" + "str s16, [x26, x28]\n" + "fmla v15.4s, v19.4s, v4.4s\n" + "fmla v18.4s, v31.4s, v4.4s\n" + "ldr x26, [%[outptrs], 112]\n" + "fmla v21.4s, v19.4s, v7.4s\n" + "fmla v24.4s, v19.4s, v5.4s\n" + "fmla v20.4s, v19.4s, v8.4s\n" + "str s15, [x25, x28]\n" + "str s18, [x24, x28]\n" + "ldr x25, [%[outptrs], 88]\n" + "fmla v24.4s, v31.4s, v7.4s\n" + "fmla v21.4s, v17.4s, v4.4s\n" + "fmla v20.4s, v31.4s, v10.4s\n" + "str s21, [x26, x28]\n" + "fmla v20.4s, v17.4s, v5.4s\n" + "ldr x26, [%[outptrs], 120]\n" + "fmla v24.4s, v14.4s, v4.4s\n" + "fmla v20.4s, v14.4s, v7.4s\n" + "str s24, [x25, x28]\n" + "fmla v20.4s, v26.4s, v4.4s\n" + "str s20, [x26, x28]\n" + "add x28, x28, #4\n" + "7:\n" + : [wbptr] "+r" (weight_bias_ptr) + : [n_channels] "r" ((long) n_channels), [outptrs] "r" (outptrs), [inptrs] "r" (inptrs) + : "cc", "v0", "v1", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v2", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v3", "v30", "v31", "v4", "v5", "v6", "v7", "v8", "v9", "x15", "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "memory" + ); +} + +template <> +template <> void Conv::execute_tile<ActivationFunction::ReLU>( int n_channels, const void *weight_bias_ptr, @@ -2374,6 +3523,1223 @@ void Conv::execute_tile<ActivationFunction::ReLU>( template <> template <> +void Conv::execute_tile<ActivationFunction::ReLU>( + int n_channels, + const void *weight_bias_ptr, + const float *inptrs[6][6], + float *outptrs[4][4] +) +{ + __asm __volatile( + "mov x27, xzr\n" + "mov x28, xzr\n" + "and x19, %[n_channels], #3\n" + "lsr x26, %[n_channels], #2\n" + "cbz x26, 4f\n" + "1:\n" + "ldr q25, [%[wbptr]]\n" + "ldr x25, [%[inptrs], 0]\n" + "mov v2.16b, v25.16b\n" + "ldr q22, [%[wbptr], #16]\n" + "mov v16.16b, v25.16b\n" + "ldr q9, [%[wbptr], #32]\n" + "mov v18.16b, v25.16b\n" + "ldr q8, [%[wbptr], #48]\n" + "mov v13.16b, v25.16b\n" + "ldr q19, [%[wbptr], #64]\n" + "mov v0.16b, v25.16b\n" + "ldr q7, [%[wbptr], #80]\n" + "mov v17.16b, v25.16b\n" + "ldr q6, [%[wbptr], #96]\n" + "mov v14.16b, v25.16b\n" + "ldr q5, [%[wbptr], #112]\n" + "mov v12.16b, v25.16b\n" + "ldr q4, [%[wbptr], #128]\n" + "mov v15.16b, v25.16b\n" + "ldr q3, [%[wbptr], #144]\n" + "ldr q27, [x25, x27]\n" + "ldr x17, [%[inptrs], 48]\n" + "fmla v2.4s, v27.4s, v22.4s\n" + "ldr x25, [%[inptrs], 8]\n" + "ldr q26, [x17, x27]\n" + "ldr x24, [%[inptrs], 96]\n" + "fmla v16.4s, v26.4s, v22.4s\n" + "ldr q31, [x25, x27]\n" + "ldr q28, [x24, x27]\n" + "ldr x17, [%[inptrs], 56]\n" + "fmla v2.4s, v26.4s, v19.4s\n" + "ldr x25, [%[inptrs], 16]\n" + "ldr q29, [x17, x27]\n" + "ldr x18, [%[inptrs], 144]\n" + "ldr x24, [%[inptrs], 104]\n" + "subs x26, x26, #1\n" + "ldr q30, [x25, x27]\n" + "ldr q27, [x18, x27]\n" + "ldr q21, [x24, x27]\n" + "fmla v2.4s, v31.4s, v9.4s\n" + "beq 3f\n" + "2:\n" + "mov v1.16b, v25.16b\n" + "ldr x17, [%[inptrs], 64]\n" + "mov v10.16b, v25.16b\n" + "ldr x25, [%[inptrs], 24]\n" + "fmla v18.4s, v31.4s, v22.4s\n" + "ldr q23, [x17, x27]\n" + "fmla v2.4s, v28.4s, v5.4s\n" + "ldr x15, [%[inptrs], 192]\n" + "fmla v16.4s, v28.4s, v19.4s\n" + "ldr x18, [%[inptrs], 152]\n" + "fmla v13.4s, v28.4s, v22.4s\n" + "ldr q26, [x25, x27]\n" + "fmla v18.4s, v29.4s, v19.4s\n" + "ldr x24, [%[inptrs], 112]\n" + "fmla v2.4s, v29.4s, v7.4s\n" + "ldr x17, [%[inptrs], 72]\n" + "fmla v16.4s, v29.4s, v9.4s\n" + "ldr x25, [%[inptrs], 32]\n" + "fmla v0.4s, v29.4s, v22.4s\n" + "ldr q28, [x15, x27]\n" + "fmla v18.4s, v30.4s, v9.4s\n" + "ldr x16, [%[inptrs], 240]\n" + "fmla v2.4s, v30.4s, v8.4s\n" + "ldr x15, [%[inptrs], 200]\n" + "fmla v17.4s, v30.4s, v22.4s\n" + "ldr q29, [x18, x27]\n" + "fmla v16.4s, v27.4s, v5.4s\n" + "ldr x18, [%[inptrs], 160]\n" + "fmla v13.4s, v27.4s, v19.4s\n" + "ldr x20, [%[outptrs], 0]\n" + "fmla v14.4s, v27.4s, v22.4s\n" + "ldr q20, [x24, x27]\n" + "fmla v2.4s, v21.4s, v4.4s\n" + "ldr x24, [%[inptrs], 120]\n" + "fmla v16.4s, v21.4s, v7.4s\n" + "ldr x21, [%[outptrs], 32]\n" + "fmla v18.4s, v21.4s, v5.4s\n" + "ldr x22, [%[outptrs], 64]\n" + "fmla v13.4s, v21.4s, v9.4s\n" + "ldr x23, [%[outptrs], 96]\n" + "fmla v0.4s, v21.4s, v19.4s\n" + "add %[wbptr], %[wbptr], #160\n" + "fmla v12.4s, v21.4s, v22.4s\n" + "ldr q24, [x17, x27]\n" + "fmla v2.4s, v23.4s, v6.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v16.4s, v23.4s, v8.4s\n" + "ldr x17, [%[inptrs], 80]\n" + "fmla v18.4s, v23.4s, v7.4s\n" + "subs x26, x26, #1\n" + "fmla v0.4s, v23.4s, v9.4s\n" + "fmla v17.4s, v23.4s, v19.4s\n" + "fmla v15.4s, v23.4s, v22.4s\n" + "ldr q23, [x25, x27]\n" + "fmla v1.4s, v26.4s, v22.4s\n" + "ldr x25, [%[inptrs], 40]\n" + "fmla v18.4s, v26.4s, v8.4s\n" + "fmla v13.4s, v28.4s, v5.4s\n" + "fmla v17.4s, v26.4s, v9.4s\n" + "ldr q30, [x16, x27]\n" + "fmla v14.4s, v28.4s, v19.4s\n" + "ldr q26, [x15, x27]\n" + "fmla v16.4s, v29.4s, v4.4s\n" + "ldr x16, [%[inptrs], 248]\n" + "fmla v13.4s, v29.4s, v7.4s\n" + "ldr x15, [%[inptrs], 208]\n" + "fmla v0.4s, v29.4s, v5.4s\n" + "fmla v12.4s, v29.4s, v19.4s\n" + "fmla v14.4s, v29.4s, v9.4s\n" + "fmla v10.4s, v29.4s, v22.4s\n" + "mov v11.16b, v25.16b\n" + "fmla v2.4s, v20.4s, v3.4s\n" + "fmla v16.4s, v20.4s, v6.4s\n" + "fmla v18.4s, v20.4s, v4.4s\n" + "fmla v13.4s, v20.4s, v8.4s\n" + "fmla v0.4s, v20.4s, v7.4s\n" + "fmla v17.4s, v20.4s, v5.4s\n" + "fmla v12.4s, v20.4s, v9.4s\n" + "fmla v15.4s, v20.4s, v19.4s\n" + "fmla v11.4s, v20.4s, v22.4s\n" + "mov v21.16b, v25.16b\n" + "fmla v18.4s, v24.4s, v6.4s\n" + "fmla v0.4s, v24.4s, v8.4s\n" + "fmla v1.4s, v24.4s, v19.4s\n" + "fmla v17.4s, v24.4s, v7.4s\n" + "fmla v14.4s, v30.4s, v5.4s\n" + "mov v20.16b, v25.16b\n" + "fmla v15.4s, v24.4s, v9.4s\n" + "fmla v21.4s, v24.4s, v22.4s\n" + "ldr q27, [x18, x27]\n" + "fmla v1.4s, v23.4s, v9.4s\n" + "ldr x18, [%[inptrs], 168]\n" + "fmla v17.4s, v23.4s, v8.4s\n" + "ldr q30, [x24, x27]\n" + "fmla v13.4s, v26.4s, v4.4s\n" + "ldr x24, [%[inptrs], 128]\n" + "fmla v14.4s, v26.4s, v7.4s\n" + "fmla v12.4s, v26.4s, v5.4s\n" + "fmla v10.4s, v26.4s, v19.4s\n" + "ldr q31, [x17, x27]\n" + "fmla v16.4s, v27.4s, v3.4s\n" + "ldr x17, [%[inptrs], 88]\n" + "fmla v13.4s, v27.4s, v6.4s\n" + "fmla v0.4s, v27.4s, v4.4s\n" + "fmla v14.4s, v27.4s, v8.4s\n" + "fmla v12.4s, v27.4s, v7.4s\n" + "fmla v15.4s, v27.4s, v5.4s\n" + "fmla v10.4s, v27.4s, v9.4s\n" + "fmla v11.4s, v27.4s, v19.4s\n" + "fmla v20.4s, v27.4s, v22.4s\n" + "mov v24.16b, v25.16b\n" + "mov v23.16b, v25.16b\n" + "fmla v18.4s, v30.4s, v3.4s\n" + "fmla v0.4s, v30.4s, v6.4s\n" + "fmla v17.4s, v30.4s, v4.4s\n" + "fmla v12.4s, v30.4s, v8.4s\n" + "fmla v15.4s, v30.4s, v7.4s\n" + "fmla v1.4s, v30.4s, v5.4s\n" + "fmla v11.4s, v30.4s, v9.4s\n" + "fmla v21.4s, v30.4s, v19.4s\n" + "fmla v24.4s, v30.4s, v22.4s\n" + "ldr q25, [x25, x27]\n" + "fmla v17.4s, v31.4s, v6.4s\n" + "ldr x25, [%[inptrs], 0]\n" + "fmla v15.4s, v31.4s, v8.4s\n" + "fmla v1.4s, v31.4s, v7.4s\n" + "fmla v21.4s, v31.4s, v9.4s\n" + "ldr q26, [x16, x27]\n" + "fmla v14.4s, v26.4s, v4.4s\n" + "ldr x16, [%[inptrs], 256]\n" + "fmla v10.4s, v26.4s, v5.4s\n" + "ldr q31, [x15, x27]\n" + "fmla v1.4s, v25.4s, v8.4s\n" + "ldr q29, [x18, x27]\n" + "fmla v13.4s, v31.4s, v3.4s\n" + "ldr x15, [%[inptrs], 216]\n" + "fmla v14.4s, v31.4s, v6.4s\n" + "ldr x18, [%[inptrs], 176]\n" + "fmla v12.4s, v31.4s, v4.4s\n" + "fmla v10.4s, v31.4s, v7.4s\n" + "fmla v11.4s, v31.4s, v5.4s\n" + "fmla v20.4s, v31.4s, v19.4s\n" + "fmla v0.4s, v29.4s, v3.4s\n" + "ldr q28, [x24, x27]\n" + "fmla v15.4s, v29.4s, v4.4s\n" + "ldr x24, [%[inptrs], 136]\n" + "fmla v12.4s, v29.4s, v6.4s\n" + "fmla v10.4s, v29.4s, v8.4s\n" + "fmla v11.4s, v29.4s, v7.4s\n" + "fmla v21.4s, v29.4s, v5.4s\n" + "fmla v20.4s, v29.4s, v9.4s\n" + "fmla v24.4s, v29.4s, v19.4s\n" + "fmla v23.4s, v29.4s, v22.4s\n" + "ldr q25, [x17, x27]\n" + "fmla v17.4s, v28.4s, v3.4s\n" + "ldr q29, [x16, x27]\n" + "fmla v15.4s, v28.4s, v6.4s\n" + "ldr x16, [%[inptrs], 264]\n" + "fmla v1.4s, v28.4s, v4.4s\n" + "ldr x17, [%[inptrs], 48]\n" + "fmla v11.4s, v28.4s, v8.4s\n" + "fmla v21.4s, v28.4s, v7.4s\n" + "fmla v24.4s, v28.4s, v9.4s\n" + "ldr q22, [x15, x27]\n" + "fmla v14.4s, v29.4s, v3.4s\n" + "ldr x15, [%[inptrs], 224]\n" + "fmla v1.4s, v25.4s, v6.4s\n" + "fmla v10.4s, v29.4s, v4.4s\n" + "fmla v21.4s, v25.4s, v8.4s\n" + "ldr q27, [x18, x27]\n" + "fmla v20.4s, v29.4s, v5.4s\n" + "ldr q26, [x24, x27]\n" + "fmla v12.4s, v22.4s, v3.4s\n" + "ldr x18, [%[inptrs], 184]\n" + "fmla v10.4s, v22.4s, v6.4s\n" + "ldr x24, [%[inptrs], 96]\n" + "fmla v11.4s, v22.4s, v4.4s\n" + "fmla v24.4s, v22.4s, v5.4s\n" + "fmla v20.4s, v22.4s, v7.4s\n" + "fmla v23.4s, v22.4s, v19.4s\n" + "fmla v15.4s, v27.4s, v3.4s\n" + "ldr q25, [x16, x27]\n" + "fmla v21.4s, v27.4s, v4.4s\n" + "ldr q31, [x15, x27]\n" + "fmla v11.4s, v27.4s, v6.4s\n" + "ldr x16, [%[inptrs], 272]\n" + "fmla v20.4s, v27.4s, v8.4s\n" + "ldr x15, [%[inptrs], 232]\n" + "fmla v24.4s, v27.4s, v7.4s\n" + "fmla v23.4s, v27.4s, v9.4s\n" + "fmla v1.4s, v26.4s, v3.4s\n" + "ldr q22, [x18, x27]\n" + "fmla v21.4s, v26.4s, v6.4s\n" + "ldr q19, [x16, x27]\n" + "fmla v10.4s, v25.4s, v3.4s\n" + "ldr x16, [%[inptrs], 280]\n" + "fmla v24.4s, v26.4s, v8.4s\n" + "ldr q28, [x15, x27]\n" + "fmla v20.4s, v25.4s, v4.4s\n" + "ldr x18, [%[inptrs], 144]\n" + "fmla v23.4s, v25.4s, v5.4s\n" + "ldr q30, [x16, x27]\n" + "fmla v11.4s, v31.4s, v3.4s\n" + "add x27, x27, #16\n" + "fmla v24.4s, v31.4s, v4.4s\n" + "ldr q27, [x25, x27]\n" + "fmla v20.4s, v31.4s, v6.4s\n" + "ldr x25, [%[inptrs], 8]\n" + "fmla v23.4s, v31.4s, v7.4s\n" + "movi v29.16b, #0\n" + "fmla v21.4s, v22.4s, v3.4s\n" + "ldr q26, [x17, x27]\n" + "fmla v24.4s, v22.4s, v6.4s\n" + "ldr x17, [%[inptrs], 56]\n" + "fmla v20.4s, v19.4s, v3.4s\n" + "fmax v2.4s, v2.4s, v29.4s\n" + "fmla v23.4s, v22.4s, v8.4s\n" + "ldr q25, [%[wbptr]]\n" + "fmax v18.4s, v18.4s, v29.4s\n" + "ldr q22, [%[wbptr], #16]\n" + "str q2, [x20, x28]\n" + "fmla v24.4s, v28.4s, v3.4s\n" + "fmax v17.4s, v17.4s, v29.4s\n" + "ldr q9, [%[wbptr], #32]\n" + "fmla v23.4s, v19.4s, v4.4s\n" + "ldr q8, [%[wbptr], #48]\n" + "fmax v1.4s, v1.4s, v29.4s\n" + "ldr q19, [%[wbptr], #64]\n" + "fmax v16.4s, v16.4s, v29.4s\n" + "ldr x20, [%[outptrs], 8]\n" + "fmax v0.4s, v0.4s, v29.4s\n" + "fmax v15.4s, v15.4s, v29.4s\n" + "str q18, [x20, x28]\n" + "fmla v23.4s, v28.4s, v6.4s\n" + "str q16, [x21, x28]\n" + "fmax v21.4s, v21.4s, v29.4s\n" + "fmax v13.4s, v13.4s, v29.4s\n" + "ldr q7, [%[wbptr], #80]\n" + "fmax v12.4s, v12.4s, v29.4s\n" + "ldr q5, [%[wbptr], #112]\n" + "fmla v23.4s, v30.4s, v3.4s\n" + "ldr q6, [%[wbptr], #96]\n" + "str q13, [x22, x28]\n" + "fmax v11.4s, v11.4s, v29.4s\n" + "fmax v24.4s, v24.4s, v29.4s\n" + "ldr q4, [%[wbptr], #128]\n" + "fmax v14.4s, v14.4s, v29.4s\n" + "ldr q31, [x25, x27]\n" + "fmax v10.4s, v10.4s, v29.4s\n" + "ldr q3, [%[wbptr], #144]\n" + "fmax v20.4s, v20.4s, v29.4s\n" + "ldr q28, [x24, x27]\n" + "str q14, [x23, x28]\n" + "fmax v23.4s, v23.4s, v29.4s\n" + "mov v2.16b, v25.16b\n" + "ldr q29, [x17, x27]\n" + "ldr x20, [%[outptrs], 16]\n" + "ldr x21, [%[outptrs], 40]\n" + "ldr x22, [%[outptrs], 72]\n" + "ldr x23, [%[outptrs], 104]\n" + "ldr x25, [%[inptrs], 16]\n" + "ldr x24, [%[inptrs], 104]\n" + "str q17, [x20, x28]\n" + "mov v16.16b, v25.16b\n" + "str q0, [x21, x28]\n" + "mov v18.16b, v25.16b\n" + "str q12, [x22, x28]\n" + "mov v13.16b, v25.16b\n" + "str q10, [x23, x28]\n" + "mov v0.16b, v25.16b\n" + "fmla v2.4s, v27.4s, v22.4s\n" + "ldr q30, [x25, x27]\n" + "fmla v16.4s, v26.4s, v22.4s\n" + "ldr x20, [%[outptrs], 24]\n" + "mov v17.16b, v25.16b\n" + "ldr x21, [%[outptrs], 48]\n" + "str q1, [x20, x28]\n" + "mov v14.16b, v25.16b\n" + "str q15, [x21, x28]\n" + "mov v12.16b, v25.16b\n" + "mov v15.16b, v25.16b\n" + "ldr x21, [%[outptrs], 56]\n" + "fmla v2.4s, v26.4s, v19.4s\n" + "ldr q27, [x18, x27]\n" + "str q21, [x21, x28]\n" + "ldr x22, [%[outptrs], 80]\n" + "ldr q21, [x24, x27]\n" + "ldr x23, [%[outptrs], 112]\n" + "str q11, [x22, x28]\n" + "fmla v2.4s, v31.4s, v9.4s\n" + "str q20, [x23, x28]\n" + "ldr x22, [%[outptrs], 88]\n" + "ldr x23, [%[outptrs], 120]\n" + "str q24, [x22, x28]\n" + "str q23, [x23, x28]\n" + "add x28, x28, #16\n" + "bne 2b\n" + "3:\n" + "mov v1.16b, v25.16b\n" + "ldr x17, [%[inptrs], 64]\n" + "mov v10.16b, v25.16b\n" + "ldr x25, [%[inptrs], 24]\n" + "mov v11.16b, v25.16b\n" + "ldr x15, [%[inptrs], 192]\n" + "fmla v18.4s, v31.4s, v22.4s\n" + "ldr q23, [x17, x27]\n" + "fmla v2.4s, v28.4s, v5.4s\n" + "ldr x18, [%[inptrs], 152]\n" + "fmla v16.4s, v28.4s, v19.4s\n" + "ldr x24, [%[inptrs], 112]\n" + "fmla v13.4s, v28.4s, v22.4s\n" + "ldr q26, [x25, x27]\n" + "fmla v18.4s, v29.4s, v19.4s\n" + "ldr x17, [%[inptrs], 72]\n" + "fmla v2.4s, v29.4s, v7.4s\n" + "ldr x25, [%[inptrs], 32]\n" + "fmla v16.4s, v29.4s, v9.4s\n" + "ldr x16, [%[inptrs], 240]\n" + "fmla v0.4s, v29.4s, v22.4s\n" + "ldr q28, [x15, x27]\n" + "fmla v18.4s, v30.4s, v9.4s\n" + "ldr x15, [%[inptrs], 200]\n" + "fmla v2.4s, v30.4s, v8.4s\n" + "ldr x20, [%[outptrs], 0]\n" + "fmla v17.4s, v30.4s, v22.4s\n" + "ldr q29, [x18, x27]\n" + "fmla v16.4s, v27.4s, v5.4s\n" + "ldr x18, [%[inptrs], 160]\n" + "fmla v13.4s, v27.4s, v19.4s\n" + "ldr x21, [%[outptrs], 32]\n" + "fmla v14.4s, v27.4s, v22.4s\n" + "ldr q20, [x24, x27]\n" + "fmla v2.4s, v21.4s, v4.4s\n" + "ldr x24, [%[inptrs], 120]\n" + "fmla v16.4s, v21.4s, v7.4s\n" + "ldr x22, [%[outptrs], 64]\n" + "fmla v18.4s, v21.4s, v5.4s\n" + "ldr x23, [%[outptrs], 96]\n" + "fmla v13.4s, v21.4s, v9.4s\n" + "add %[wbptr], %[wbptr], #160\n" + "fmla v0.4s, v21.4s, v19.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v12.4s, v21.4s, v22.4s\n" + "ldr q24, [x17, x27]\n" + "fmla v2.4s, v23.4s, v6.4s\n" + "ldr x17, [%[inptrs], 80]\n" + "fmla v16.4s, v23.4s, v8.4s\n" + "fmla v18.4s, v23.4s, v7.4s\n" + "fmla v0.4s, v23.4s, v9.4s\n" + "fmla v17.4s, v23.4s, v19.4s\n" + "fmla v15.4s, v23.4s, v22.4s\n" + "ldr q23, [x25, x27]\n" + "fmla v1.4s, v26.4s, v22.4s\n" + "ldr x25, [%[inptrs], 40]\n" + "fmla v18.4s, v26.4s, v8.4s\n" + "fmla v13.4s, v28.4s, v5.4s\n" + "fmla v17.4s, v26.4s, v9.4s\n" + "ldr q30, [x16, x27]\n" + "fmla v14.4s, v28.4s, v19.4s\n" + "ldr q26, [x15, x27]\n" + "fmla v16.4s, v29.4s, v4.4s\n" + "ldr x16, [%[inptrs], 248]\n" + "fmla v13.4s, v29.4s, v7.4s\n" + "ldr x15, [%[inptrs], 208]\n" + "fmla v0.4s, v29.4s, v5.4s\n" + "fmla v12.4s, v29.4s, v19.4s\n" + "fmla v14.4s, v29.4s, v9.4s\n" + "fmla v10.4s, v29.4s, v22.4s\n" + "mov v21.16b, v25.16b\n" + "fmla v2.4s, v20.4s, v3.4s\n" + "fmla v16.4s, v20.4s, v6.4s\n" + "fmla v18.4s, v20.4s, v4.4s\n" + "fmla v13.4s, v20.4s, v8.4s\n" + "fmla v0.4s, v20.4s, v7.4s\n" + "fmla v17.4s, v20.4s, v5.4s\n" + "fmla v12.4s, v20.4s, v9.4s\n" + "fmla v15.4s, v20.4s, v19.4s\n" + "fmla v11.4s, v20.4s, v22.4s\n" + "mov v20.16b, v25.16b\n" + "fmla v18.4s, v24.4s, v6.4s\n" + "fmla v0.4s, v24.4s, v8.4s\n" + "fmla v1.4s, v24.4s, v19.4s\n" + "fmla v17.4s, v24.4s, v7.4s\n" + "fmla v21.4s, v24.4s, v22.4s\n" + "fmla v15.4s, v24.4s, v9.4s\n" + "ldr q27, [x18, x27]\n" + "fmla v14.4s, v30.4s, v5.4s\n" + "ldr q30, [x24, x27]\n" + "fmla v1.4s, v23.4s, v9.4s\n" + "ldr x18, [%[inptrs], 168]\n" + "fmla v17.4s, v23.4s, v8.4s\n" + "ldr q31, [x17, x27]\n" + "fmla v13.4s, v26.4s, v4.4s\n" + "ldr x24, [%[inptrs], 128]\n" + "fmla v14.4s, v26.4s, v7.4s\n" + "ldr x17, [%[inptrs], 88]\n" + "fmla v12.4s, v26.4s, v5.4s\n" + "fmla v10.4s, v26.4s, v19.4s\n" + "mov v24.16b, v25.16b\n" + "mov v23.16b, v25.16b\n" + "fmla v16.4s, v27.4s, v3.4s\n" + "fmla v13.4s, v27.4s, v6.4s\n" + "fmla v0.4s, v27.4s, v4.4s\n" + "fmla v14.4s, v27.4s, v8.4s\n" + "fmla v12.4s, v27.4s, v7.4s\n" + "fmla v15.4s, v27.4s, v5.4s\n" + "fmla v10.4s, v27.4s, v9.4s\n" + "fmla v11.4s, v27.4s, v19.4s\n" + "fmla v20.4s, v27.4s, v22.4s\n" + "ldr q25, [x25, x27]\n" + "fmla v18.4s, v30.4s, v3.4s\n" + "fmla v0.4s, v30.4s, v6.4s\n" + "fmla v17.4s, v30.4s, v4.4s\n" + "fmla v12.4s, v30.4s, v8.4s\n" + "fmla v15.4s, v30.4s, v7.4s\n" + "fmla v1.4s, v30.4s, v5.4s\n" + "fmla v11.4s, v30.4s, v9.4s\n" + "fmla v21.4s, v30.4s, v19.4s\n" + "fmla v24.4s, v30.4s, v22.4s\n" + "ldr q26, [x16, x27]\n" + "fmla v17.4s, v31.4s, v6.4s\n" + "ldr x16, [%[inptrs], 256]\n" + "fmla v15.4s, v31.4s, v8.4s\n" + "fmla v1.4s, v31.4s, v7.4s\n" + "fmla v21.4s, v31.4s, v9.4s\n" + "ldr q31, [x15, x27]\n" + "fmla v14.4s, v26.4s, v4.4s\n" + "ldr x15, [%[inptrs], 216]\n" + "fmla v10.4s, v26.4s, v5.4s\n" + "ldr q29, [x18, x27]\n" + "fmla v1.4s, v25.4s, v8.4s\n" + "ldr q28, [x24, x27]\n" + "fmla v13.4s, v31.4s, v3.4s\n" + "ldr x18, [%[inptrs], 176]\n" + "fmla v14.4s, v31.4s, v6.4s\n" + "ldr x24, [%[inptrs], 136]\n" + "fmla v12.4s, v31.4s, v4.4s\n" + "fmla v10.4s, v31.4s, v7.4s\n" + "fmla v11.4s, v31.4s, v5.4s\n" + "fmla v20.4s, v31.4s, v19.4s\n" + "fmla v0.4s, v29.4s, v3.4s\n" + "ldr q25, [x17, x27]\n" + "fmla v15.4s, v29.4s, v4.4s\n" + "fmla v21.4s, v29.4s, v5.4s\n" + "fmla v12.4s, v29.4s, v6.4s\n" + "fmla v10.4s, v29.4s, v8.4s\n" + "fmla v11.4s, v29.4s, v7.4s\n" + "fmla v20.4s, v29.4s, v9.4s\n" + "fmla v24.4s, v29.4s, v19.4s\n" + "fmla v23.4s, v29.4s, v22.4s\n" + "fmla v17.4s, v28.4s, v3.4s\n" + "ldr q29, [x16, x27]\n" + "fmla v15.4s, v28.4s, v6.4s\n" + "ldr q22, [x15, x27]\n" + "fmla v1.4s, v28.4s, v4.4s\n" + "ldr x16, [%[inptrs], 264]\n" + "fmla v11.4s, v28.4s, v8.4s\n" + "ldr x15, [%[inptrs], 224]\n" + "fmla v21.4s, v28.4s, v7.4s\n" + "fmla v24.4s, v28.4s, v9.4s\n" + "fmla v14.4s, v29.4s, v3.4s\n" + "ldr q27, [x18, x27]\n" + "fmla v1.4s, v25.4s, v6.4s\n" + "ldr x18, [%[inptrs], 184]\n" + "fmla v10.4s, v29.4s, v4.4s\n" + "fmla v20.4s, v29.4s, v5.4s\n" + "fmla v21.4s, v25.4s, v8.4s\n" + "ldr q26, [x24, x27]\n" + "fmla v12.4s, v22.4s, v3.4s\n" + "ldr q25, [x16, x27]\n" + "fmla v11.4s, v22.4s, v4.4s\n" + "ldr x16, [%[inptrs], 272]\n" + "fmla v10.4s, v22.4s, v6.4s\n" + "fmla v20.4s, v22.4s, v7.4s\n" + "fmla v24.4s, v22.4s, v5.4s\n" + "fmla v23.4s, v22.4s, v19.4s\n" + "fmla v15.4s, v27.4s, v3.4s\n" + "ldr q31, [x15, x27]\n" + "fmla v11.4s, v27.4s, v6.4s\n" + "ldr q22, [x18, x27]\n" + "fmla v21.4s, v27.4s, v4.4s\n" + "ldr x15, [%[inptrs], 232]\n" + "fmla v20.4s, v27.4s, v8.4s\n" + "fmla v24.4s, v27.4s, v7.4s\n" + "fmla v23.4s, v27.4s, v9.4s\n" + "ldr q19, [x16, x27]\n" + "fmla v1.4s, v26.4s, v3.4s\n" + "ldr q28, [x15, x27]\n" + "fmla v21.4s, v26.4s, v6.4s\n" + "ldr x16, [%[inptrs], 280]\n" + "fmla v24.4s, v26.4s, v8.4s\n" + "fmla v10.4s, v25.4s, v3.4s\n" + "fmla v20.4s, v25.4s, v4.4s\n" + "ldr q30, [x16, x27]\n" + "fmla v23.4s, v25.4s, v5.4s\n" + "add x27, x27, #16\n" + "fmla v11.4s, v31.4s, v3.4s\n" + "fmla v21.4s, v22.4s, v3.4s\n" + "fmla v24.4s, v31.4s, v4.4s\n" + "movi v29.16b, #0\n" + "fmla v20.4s, v31.4s, v6.4s\n" + "fmla v23.4s, v31.4s, v7.4s\n" + "fmax v2.4s, v2.4s, v29.4s\n" + "fmax v18.4s, v18.4s, v29.4s\n" + "fmla v24.4s, v22.4s, v6.4s\n" + "fmax v17.4s, v17.4s, v29.4s\n" + "fmla v20.4s, v19.4s, v3.4s\n" + "fmax v1.4s, v1.4s, v29.4s\n" + "str q2, [x20, x28]\n" + "fmla v23.4s, v22.4s, v8.4s\n" + "fmax v16.4s, v16.4s, v29.4s\n" + "ldr x20, [%[outptrs], 8]\n" + "fmla v24.4s, v28.4s, v3.4s\n" + "fmax v0.4s, v0.4s, v29.4s\n" + "str q18, [x20, x28]\n" + "fmax v15.4s, v15.4s, v29.4s\n" + "str q16, [x21, x28]\n" + "fmla v23.4s, v19.4s, v4.4s\n" + "fmax v21.4s, v21.4s, v29.4s\n" + "ldr x20, [%[outptrs], 16]\n" + "fmax v13.4s, v13.4s, v29.4s\n" + "ldr x21, [%[outptrs], 40]\n" + "str q17, [x20, x28]\n" + "fmax v12.4s, v12.4s, v29.4s\n" + "str q0, [x21, x28]\n" + "fmla v23.4s, v28.4s, v6.4s\n" + "str q13, [x22, x28]\n" + "fmax v11.4s, v11.4s, v29.4s\n" + "fmax v24.4s, v24.4s, v29.4s\n" + "ldr x20, [%[outptrs], 24]\n" + "fmax v14.4s, v14.4s, v29.4s\n" + "ldr x21, [%[outptrs], 48]\n" + "str q1, [x20, x28]\n" + "fmla v23.4s, v30.4s, v3.4s\n" + "str q15, [x21, x28]\n" + "fmax v10.4s, v10.4s, v29.4s\n" + "str q14, [x23, x28]\n" + "fmax v20.4s, v20.4s, v29.4s\n" + "ldr x21, [%[outptrs], 56]\n" + "ldr x22, [%[outptrs], 72]\n" + "ldr x23, [%[outptrs], 104]\n" + "fmax v23.4s, v23.4s, v29.4s\n" + "str q21, [x21, x28]\n" + "str q12, [x22, x28]\n" + "str q10, [x23, x28]\n" + "ldr x22, [%[outptrs], 80]\n" + "ldr x23, [%[outptrs], 112]\n" + "str q11, [x22, x28]\n" + "str q20, [x23, x28]\n" + "ldr x22, [%[outptrs], 88]\n" + "ldr x23, [%[outptrs], 120]\n" + "str q24, [x22, x28]\n" + "str q23, [x23, x28]\n" + "add x28, x28, #16\n" + "4:\n" + "cbz x19, 7f\n" + "ldr s25, [%[wbptr]]\n" + "mov v2.16b, v25.16b\n" + "ldr s22, [%[wbptr], #4]\n" + "mov v16.16b, v25.16b\n" + "ldr s9, [%[wbptr], #8]\n" + "mov v18.16b, v25.16b\n" + "ldr s8, [%[wbptr], #12]\n" + "mov v13.16b, v25.16b\n" + "ldr s19, [%[wbptr], #16]\n" + "mov v0.16b, v25.16b\n" + "ldr s7, [%[wbptr], #20]\n" + "mov v17.16b, v25.16b\n" + "ldr s6, [%[wbptr], #24]\n" + "mov v14.16b, v25.16b\n" + "ldr s5, [%[wbptr], #28]\n" + "mov v12.16b, v25.16b\n" + "ldr s4, [%[wbptr], #32]\n" + "mov v15.16b, v25.16b\n" + "ldr s3, [%[wbptr], #36]\n" + "ldr x25, [%[inptrs], 0]\n" + "ldr x17, [%[inptrs], 48]\n" + "ldr x24, [%[inptrs], 96]\n" + "ldr x18, [%[inptrs], 144]\n" + "subs x19, x19, #1\n" + "ldr s27, [x25, x27]\n" + "fmla v2.4s, v27.4s, v22.4s\n" + "ldr s26, [x17, x27]\n" + "fmla v16.4s, v26.4s, v22.4s\n" + "ldr s28, [x24, x27]\n" + "ldr s27, [x18, x27]\n" + "ldr x25, [%[inptrs], 8]\n" + "ldr x17, [%[inptrs], 56]\n" + "ldr x24, [%[inptrs], 104]\n" + "ldr s31, [x25, x27]\n" + "fmla v2.4s, v26.4s, v19.4s\n" + "ldr s29, [x17, x27]\n" + "ldr s21, [x24, x27]\n" + "ldr x25, [%[inptrs], 16]\n" + "ldr s30, [x25, x27]\n" + "fmla v2.4s, v31.4s, v9.4s\n" + "beq 6f\n" + "5:\n" + "mov v1.16b, v25.16b\n" + "ldr x17, [%[inptrs], 64]\n" + "mov v10.16b, v25.16b\n" + "ldr x25, [%[inptrs], 24]\n" + "fmla v18.4s, v31.4s, v22.4s\n" + "ldr s23, [x17, x27]\n" + "fmla v2.4s, v28.4s, v5.4s\n" + "ldr x15, [%[inptrs], 192]\n" + "fmla v16.4s, v28.4s, v19.4s\n" + "ldr x18, [%[inptrs], 152]\n" + "fmla v13.4s, v28.4s, v22.4s\n" + "ldr s26, [x25, x27]\n" + "fmla v18.4s, v29.4s, v19.4s\n" + "ldr x24, [%[inptrs], 112]\n" + "fmla v2.4s, v29.4s, v7.4s\n" + "ldr x17, [%[inptrs], 72]\n" + "fmla v16.4s, v29.4s, v9.4s\n" + "ldr x25, [%[inptrs], 32]\n" + "fmla v0.4s, v29.4s, v22.4s\n" + "ldr s28, [x15, x27]\n" + "fmla v18.4s, v30.4s, v9.4s\n" + "ldr x16, [%[inptrs], 240]\n" + "fmla v2.4s, v30.4s, v8.4s\n" + "ldr x15, [%[inptrs], 200]\n" + "fmla v17.4s, v30.4s, v22.4s\n" + "ldr s29, [x18, x27]\n" + "fmla v16.4s, v27.4s, v5.4s\n" + "ldr x18, [%[inptrs], 160]\n" + "fmla v13.4s, v27.4s, v19.4s\n" + "ldr x20, [%[outptrs], 0]\n" + "fmla v14.4s, v27.4s, v22.4s\n" + "ldr s20, [x24, x27]\n" + "fmla v2.4s, v21.4s, v4.4s\n" + "ldr x24, [%[inptrs], 120]\n" + "fmla v16.4s, v21.4s, v7.4s\n" + "ldr x21, [%[outptrs], 32]\n" + "fmla v18.4s, v21.4s, v5.4s\n" + "ldr x22, [%[outptrs], 64]\n" + "fmla v13.4s, v21.4s, v9.4s\n" + "ldr x23, [%[outptrs], 96]\n" + "fmla v0.4s, v21.4s, v19.4s\n" + "add %[wbptr], %[wbptr], #40\n" + "fmla v12.4s, v21.4s, v22.4s\n" + "ldr s24, [x17, x27]\n" + "fmla v2.4s, v23.4s, v6.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v16.4s, v23.4s, v8.4s\n" + "ldr x17, [%[inptrs], 80]\n" + "fmla v18.4s, v23.4s, v7.4s\n" + "subs x19, x19, #1\n" + "fmla v0.4s, v23.4s, v9.4s\n" + "fmla v17.4s, v23.4s, v19.4s\n" + "fmla v15.4s, v23.4s, v22.4s\n" + "ldr s23, [x25, x27]\n" + "fmla v1.4s, v26.4s, v22.4s\n" + "ldr x25, [%[inptrs], 40]\n" + "fmla v18.4s, v26.4s, v8.4s\n" + "fmla v13.4s, v28.4s, v5.4s\n" + "fmla v17.4s, v26.4s, v9.4s\n" + "ldr s30, [x16, x27]\n" + "fmla v14.4s, v28.4s, v19.4s\n" + "ldr s26, [x15, x27]\n" + "fmla v16.4s, v29.4s, v4.4s\n" + "ldr x16, [%[inptrs], 248]\n" + "fmla v13.4s, v29.4s, v7.4s\n" + "ldr x15, [%[inptrs], 208]\n" + "fmla v0.4s, v29.4s, v5.4s\n" + "fmla v12.4s, v29.4s, v19.4s\n" + "fmla v14.4s, v29.4s, v9.4s\n" + "fmla v10.4s, v29.4s, v22.4s\n" + "mov v11.16b, v25.16b\n" + "fmla v2.4s, v20.4s, v3.4s\n" + "fmla v16.4s, v20.4s, v6.4s\n" + "fmla v18.4s, v20.4s, v4.4s\n" + "fmla v13.4s, v20.4s, v8.4s\n" + "fmla v0.4s, v20.4s, v7.4s\n" + "fmla v17.4s, v20.4s, v5.4s\n" + "fmla v12.4s, v20.4s, v9.4s\n" + "fmla v15.4s, v20.4s, v19.4s\n" + "fmla v11.4s, v20.4s, v22.4s\n" + "mov v21.16b, v25.16b\n" + "fmla v18.4s, v24.4s, v6.4s\n" + "fmla v0.4s, v24.4s, v8.4s\n" + "fmla v1.4s, v24.4s, v19.4s\n" + "fmla v17.4s, v24.4s, v7.4s\n" + "fmla v14.4s, v30.4s, v5.4s\n" + "mov v20.16b, v25.16b\n" + "fmla v15.4s, v24.4s, v9.4s\n" + "fmla v21.4s, v24.4s, v22.4s\n" + "ldr s27, [x18, x27]\n" + "fmla v1.4s, v23.4s, v9.4s\n" + "ldr x18, [%[inptrs], 168]\n" + "fmla v17.4s, v23.4s, v8.4s\n" + "ldr s30, [x24, x27]\n" + "fmla v13.4s, v26.4s, v4.4s\n" + "ldr x24, [%[inptrs], 128]\n" + "fmla v14.4s, v26.4s, v7.4s\n" + "fmla v12.4s, v26.4s, v5.4s\n" + "fmla v10.4s, v26.4s, v19.4s\n" + "ldr s31, [x17, x27]\n" + "fmla v16.4s, v27.4s, v3.4s\n" + "ldr x17, [%[inptrs], 88]\n" + "fmla v13.4s, v27.4s, v6.4s\n" + "fmla v0.4s, v27.4s, v4.4s\n" + "fmla v14.4s, v27.4s, v8.4s\n" + "fmla v12.4s, v27.4s, v7.4s\n" + "fmla v15.4s, v27.4s, v5.4s\n" + "fmla v10.4s, v27.4s, v9.4s\n" + "fmla v11.4s, v27.4s, v19.4s\n" + "fmla v20.4s, v27.4s, v22.4s\n" + "mov v24.16b, v25.16b\n" + "mov v23.16b, v25.16b\n" + "fmla v18.4s, v30.4s, v3.4s\n" + "fmla v0.4s, v30.4s, v6.4s\n" + "fmla v17.4s, v30.4s, v4.4s\n" + "fmla v12.4s, v30.4s, v8.4s\n" + "fmla v15.4s, v30.4s, v7.4s\n" + "fmla v1.4s, v30.4s, v5.4s\n" + "fmla v11.4s, v30.4s, v9.4s\n" + "fmla v21.4s, v30.4s, v19.4s\n" + "fmla v24.4s, v30.4s, v22.4s\n" + "ldr s25, [x25, x27]\n" + "fmla v17.4s, v31.4s, v6.4s\n" + "ldr x25, [%[inptrs], 0]\n" + "fmla v15.4s, v31.4s, v8.4s\n" + "fmla v1.4s, v31.4s, v7.4s\n" + "fmla v21.4s, v31.4s, v9.4s\n" + "ldr s26, [x16, x27]\n" + "fmla v14.4s, v26.4s, v4.4s\n" + "ldr x16, [%[inptrs], 256]\n" + "fmla v10.4s, v26.4s, v5.4s\n" + "ldr s31, [x15, x27]\n" + "fmla v1.4s, v25.4s, v8.4s\n" + "ldr s29, [x18, x27]\n" + "fmla v13.4s, v31.4s, v3.4s\n" + "ldr x15, [%[inptrs], 216]\n" + "fmla v14.4s, v31.4s, v6.4s\n" + "ldr x18, [%[inptrs], 176]\n" + "fmla v12.4s, v31.4s, v4.4s\n" + "fmla v10.4s, v31.4s, v7.4s\n" + "fmla v11.4s, v31.4s, v5.4s\n" + "fmla v20.4s, v31.4s, v19.4s\n" + "fmla v0.4s, v29.4s, v3.4s\n" + "ldr s28, [x24, x27]\n" + "fmla v15.4s, v29.4s, v4.4s\n" + "ldr x24, [%[inptrs], 136]\n" + "fmla v12.4s, v29.4s, v6.4s\n" + "fmla v10.4s, v29.4s, v8.4s\n" + "fmla v11.4s, v29.4s, v7.4s\n" + "fmla v21.4s, v29.4s, v5.4s\n" + "fmla v20.4s, v29.4s, v9.4s\n" + "fmla v24.4s, v29.4s, v19.4s\n" + "fmla v23.4s, v29.4s, v22.4s\n" + "ldr s25, [x17, x27]\n" + "fmla v17.4s, v28.4s, v3.4s\n" + "ldr s29, [x16, x27]\n" + "fmla v15.4s, v28.4s, v6.4s\n" + "ldr x16, [%[inptrs], 264]\n" + "fmla v1.4s, v28.4s, v4.4s\n" + "ldr x17, [%[inptrs], 48]\n" + "fmla v11.4s, v28.4s, v8.4s\n" + "fmla v21.4s, v28.4s, v7.4s\n" + "fmla v24.4s, v28.4s, v9.4s\n" + "ldr s22, [x15, x27]\n" + "fmla v14.4s, v29.4s, v3.4s\n" + "ldr x15, [%[inptrs], 224]\n" + "fmla v1.4s, v25.4s, v6.4s\n" + "fmla v10.4s, v29.4s, v4.4s\n" + "fmla v21.4s, v25.4s, v8.4s\n" + "ldr s27, [x18, x27]\n" + "fmla v20.4s, v29.4s, v5.4s\n" + "ldr s26, [x24, x27]\n" + "fmla v12.4s, v22.4s, v3.4s\n" + "ldr x18, [%[inptrs], 184]\n" + "fmla v10.4s, v22.4s, v6.4s\n" + "ldr x24, [%[inptrs], 96]\n" + "fmla v11.4s, v22.4s, v4.4s\n" + "fmla v24.4s, v22.4s, v5.4s\n" + "fmla v20.4s, v22.4s, v7.4s\n" + "fmla v23.4s, v22.4s, v19.4s\n" + "fmla v15.4s, v27.4s, v3.4s\n" + "ldr s25, [x16, x27]\n" + "fmla v21.4s, v27.4s, v4.4s\n" + "ldr s31, [x15, x27]\n" + "fmla v11.4s, v27.4s, v6.4s\n" + "ldr x16, [%[inptrs], 272]\n" + "fmla v20.4s, v27.4s, v8.4s\n" + "ldr x15, [%[inptrs], 232]\n" + "fmla v24.4s, v27.4s, v7.4s\n" + "fmla v23.4s, v27.4s, v9.4s\n" + "fmla v1.4s, v26.4s, v3.4s\n" + "ldr s22, [x18, x27]\n" + "fmla v21.4s, v26.4s, v6.4s\n" + "ldr s19, [x16, x27]\n" + "fmla v10.4s, v25.4s, v3.4s\n" + "ldr x16, [%[inptrs], 280]\n" + "fmla v24.4s, v26.4s, v8.4s\n" + "ldr s28, [x15, x27]\n" + "fmla v20.4s, v25.4s, v4.4s\n" + "ldr x18, [%[inptrs], 144]\n" + "fmla v23.4s, v25.4s, v5.4s\n" + "ldr s30, [x16, x27]\n" + "fmla v11.4s, v31.4s, v3.4s\n" + "add x27, x27, #4\n" + "fmla v24.4s, v31.4s, v4.4s\n" + "ldr s27, [x25, x27]\n" + "fmla v20.4s, v31.4s, v6.4s\n" + "ldr x25, [%[inptrs], 8]\n" + "fmla v23.4s, v31.4s, v7.4s\n" + "movi v29.16b, #0\n" + "fmla v21.4s, v22.4s, v3.4s\n" + "ldr s26, [x17, x27]\n" + "fmla v24.4s, v22.4s, v6.4s\n" + "ldr x17, [%[inptrs], 56]\n" + "fmla v20.4s, v19.4s, v3.4s\n" + "fmax v2.4s, v2.4s, v29.4s\n" + "fmla v23.4s, v22.4s, v8.4s\n" + "ldr s25, [%[wbptr]]\n" + "fmax v18.4s, v18.4s, v29.4s\n" + "ldr s22, [%[wbptr], #4]\n" + "str s2, [x20, x28]\n" + "fmla v24.4s, v28.4s, v3.4s\n" + "fmax v17.4s, v17.4s, v29.4s\n" + "ldr s9, [%[wbptr], #8]\n" + "fmla v23.4s, v19.4s, v4.4s\n" + "ldr s8, [%[wbptr], #12]\n" + "fmax v1.4s, v1.4s, v29.4s\n" + "ldr s19, [%[wbptr], #16]\n" + "fmax v16.4s, v16.4s, v29.4s\n" + "ldr x20, [%[outptrs], 8]\n" + "fmax v0.4s, v0.4s, v29.4s\n" + "fmax v15.4s, v15.4s, v29.4s\n" + "str s18, [x20, x28]\n" + "fmla v23.4s, v28.4s, v6.4s\n" + "str s16, [x21, x28]\n" + "fmax v21.4s, v21.4s, v29.4s\n" + "fmax v13.4s, v13.4s, v29.4s\n" + "ldr s7, [%[wbptr], #20]\n" + "fmax v12.4s, v12.4s, v29.4s\n" + "ldr s5, [%[wbptr], #28]\n" + "fmla v23.4s, v30.4s, v3.4s\n" + "ldr s6, [%[wbptr], #24]\n" + "str s13, [x22, x28]\n" + "fmax v11.4s, v11.4s, v29.4s\n" + "fmax v24.4s, v24.4s, v29.4s\n" + "ldr s4, [%[wbptr], #32]\n" + "fmax v14.4s, v14.4s, v29.4s\n" + "ldr s31, [x25, x27]\n" + "fmax v10.4s, v10.4s, v29.4s\n" + "ldr s3, [%[wbptr], #36]\n" + "fmax v20.4s, v20.4s, v29.4s\n" + "ldr s28, [x24, x27]\n" + "str s14, [x23, x28]\n" + "fmax v23.4s, v23.4s, v29.4s\n" + "mov v2.16b, v25.16b\n" + "ldr s29, [x17, x27]\n" + "ldr x20, [%[outptrs], 16]\n" + "ldr x21, [%[outptrs], 40]\n" + "ldr x22, [%[outptrs], 72]\n" + "ldr x23, [%[outptrs], 104]\n" + "ldr x25, [%[inptrs], 16]\n" + "ldr x24, [%[inptrs], 104]\n" + "str s17, [x20, x28]\n" + "mov v16.16b, v25.16b\n" + "str s0, [x21, x28]\n" + "mov v18.16b, v25.16b\n" + "str s12, [x22, x28]\n" + "mov v13.16b, v25.16b\n" + "str s10, [x23, x28]\n" + "mov v0.16b, v25.16b\n" + "fmla v2.4s, v27.4s, v22.4s\n" + "ldr s30, [x25, x27]\n" + "fmla v16.4s, v26.4s, v22.4s\n" + "ldr x20, [%[outptrs], 24]\n" + "mov v17.16b, v25.16b\n" + "ldr x21, [%[outptrs], 48]\n" + "str s1, [x20, x28]\n" + "mov v14.16b, v25.16b\n" + "str s15, [x21, x28]\n" + "mov v12.16b, v25.16b\n" + "mov v15.16b, v25.16b\n" + "ldr x21, [%[outptrs], 56]\n" + "fmla v2.4s, v26.4s, v19.4s\n" + "ldr s27, [x18, x27]\n" + "str s21, [x21, x28]\n" + "ldr x22, [%[outptrs], 80]\n" + "ldr s21, [x24, x27]\n" + "ldr x23, [%[outptrs], 112]\n" + "str s11, [x22, x28]\n" + "fmla v2.4s, v31.4s, v9.4s\n" + "str s20, [x23, x28]\n" + "ldr x22, [%[outptrs], 88]\n" + "ldr x23, [%[outptrs], 120]\n" + "str s24, [x22, x28]\n" + "str s23, [x23, x28]\n" + "add x28, x28, #4\n" + "bne 5b\n" + "6:\n" + "mov v1.16b, v25.16b\n" + "ldr x17, [%[inptrs], 64]\n" + "mov v10.16b, v25.16b\n" + "ldr x25, [%[inptrs], 24]\n" + "mov v11.16b, v25.16b\n" + "ldr x15, [%[inptrs], 192]\n" + "fmla v18.4s, v31.4s, v22.4s\n" + "ldr s23, [x17, x27]\n" + "fmla v2.4s, v28.4s, v5.4s\n" + "ldr x18, [%[inptrs], 152]\n" + "fmla v16.4s, v28.4s, v19.4s\n" + "ldr x24, [%[inptrs], 112]\n" + "fmla v13.4s, v28.4s, v22.4s\n" + "ldr s26, [x25, x27]\n" + "fmla v18.4s, v29.4s, v19.4s\n" + "ldr x17, [%[inptrs], 72]\n" + "fmla v2.4s, v29.4s, v7.4s\n" + "ldr x25, [%[inptrs], 32]\n" + "fmla v16.4s, v29.4s, v9.4s\n" + "ldr x16, [%[inptrs], 240]\n" + "fmla v0.4s, v29.4s, v22.4s\n" + "ldr s28, [x15, x27]\n" + "fmla v18.4s, v30.4s, v9.4s\n" + "ldr x15, [%[inptrs], 200]\n" + "fmla v2.4s, v30.4s, v8.4s\n" + "ldr x20, [%[outptrs], 0]\n" + "fmla v17.4s, v30.4s, v22.4s\n" + "ldr s29, [x18, x27]\n" + "fmla v16.4s, v27.4s, v5.4s\n" + "ldr x18, [%[inptrs], 160]\n" + "fmla v13.4s, v27.4s, v19.4s\n" + "ldr x21, [%[outptrs], 32]\n" + "fmla v14.4s, v27.4s, v22.4s\n" + "ldr s20, [x24, x27]\n" + "fmla v2.4s, v21.4s, v4.4s\n" + "ldr x24, [%[inptrs], 120]\n" + "fmla v16.4s, v21.4s, v7.4s\n" + "ldr x22, [%[outptrs], 64]\n" + "fmla v18.4s, v21.4s, v5.4s\n" + "ldr x23, [%[outptrs], 96]\n" + "fmla v13.4s, v21.4s, v9.4s\n" + "add %[wbptr], %[wbptr], #40\n" + "fmla v0.4s, v21.4s, v19.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v12.4s, v21.4s, v22.4s\n" + "ldr s24, [x17, x27]\n" + "fmla v2.4s, v23.4s, v6.4s\n" + "ldr x17, [%[inptrs], 80]\n" + "fmla v16.4s, v23.4s, v8.4s\n" + "fmla v18.4s, v23.4s, v7.4s\n" + "fmla v0.4s, v23.4s, v9.4s\n" + "fmla v17.4s, v23.4s, v19.4s\n" + "fmla v15.4s, v23.4s, v22.4s\n" + "ldr s23, [x25, x27]\n" + "fmla v1.4s, v26.4s, v22.4s\n" + "ldr x25, [%[inptrs], 40]\n" + "fmla v18.4s, v26.4s, v8.4s\n" + "fmla v13.4s, v28.4s, v5.4s\n" + "fmla v17.4s, v26.4s, v9.4s\n" + "ldr s30, [x16, x27]\n" + "fmla v14.4s, v28.4s, v19.4s\n" + "ldr s26, [x15, x27]\n" + "fmla v16.4s, v29.4s, v4.4s\n" + "ldr x16, [%[inptrs], 248]\n" + "fmla v13.4s, v29.4s, v7.4s\n" + "ldr x15, [%[inptrs], 208]\n" + "fmla v0.4s, v29.4s, v5.4s\n" + "fmla v12.4s, v29.4s, v19.4s\n" + "fmla v14.4s, v29.4s, v9.4s\n" + "fmla v10.4s, v29.4s, v22.4s\n" + "mov v21.16b, v25.16b\n" + "fmla v2.4s, v20.4s, v3.4s\n" + "fmla v16.4s, v20.4s, v6.4s\n" + "fmla v18.4s, v20.4s, v4.4s\n" + "fmla v13.4s, v20.4s, v8.4s\n" + "fmla v0.4s, v20.4s, v7.4s\n" + "fmla v17.4s, v20.4s, v5.4s\n" + "fmla v12.4s, v20.4s, v9.4s\n" + "fmla v15.4s, v20.4s, v19.4s\n" + "fmla v11.4s, v20.4s, v22.4s\n" + "mov v20.16b, v25.16b\n" + "fmla v18.4s, v24.4s, v6.4s\n" + "fmla v0.4s, v24.4s, v8.4s\n" + "fmla v1.4s, v24.4s, v19.4s\n" + "fmla v17.4s, v24.4s, v7.4s\n" + "fmla v21.4s, v24.4s, v22.4s\n" + "fmla v15.4s, v24.4s, v9.4s\n" + "ldr s27, [x18, x27]\n" + "fmla v14.4s, v30.4s, v5.4s\n" + "ldr s30, [x24, x27]\n" + "fmla v1.4s, v23.4s, v9.4s\n" + "ldr x18, [%[inptrs], 168]\n" + "fmla v17.4s, v23.4s, v8.4s\n" + "ldr s31, [x17, x27]\n" + "fmla v13.4s, v26.4s, v4.4s\n" + "ldr x24, [%[inptrs], 128]\n" + "fmla v14.4s, v26.4s, v7.4s\n" + "ldr x17, [%[inptrs], 88]\n" + "fmla v12.4s, v26.4s, v5.4s\n" + "fmla v10.4s, v26.4s, v19.4s\n" + "mov v24.16b, v25.16b\n" + "mov v23.16b, v25.16b\n" + "fmla v16.4s, v27.4s, v3.4s\n" + "fmla v13.4s, v27.4s, v6.4s\n" + "fmla v0.4s, v27.4s, v4.4s\n" + "fmla v14.4s, v27.4s, v8.4s\n" + "fmla v12.4s, v27.4s, v7.4s\n" + "fmla v15.4s, v27.4s, v5.4s\n" + "fmla v10.4s, v27.4s, v9.4s\n" + "fmla v11.4s, v27.4s, v19.4s\n" + "fmla v20.4s, v27.4s, v22.4s\n" + "ldr s25, [x25, x27]\n" + "fmla v18.4s, v30.4s, v3.4s\n" + "fmla v0.4s, v30.4s, v6.4s\n" + "fmla v17.4s, v30.4s, v4.4s\n" + "fmla v12.4s, v30.4s, v8.4s\n" + "fmla v15.4s, v30.4s, v7.4s\n" + "fmla v1.4s, v30.4s, v5.4s\n" + "fmla v11.4s, v30.4s, v9.4s\n" + "fmla v21.4s, v30.4s, v19.4s\n" + "fmla v24.4s, v30.4s, v22.4s\n" + "ldr s26, [x16, x27]\n" + "fmla v17.4s, v31.4s, v6.4s\n" + "ldr x16, [%[inptrs], 256]\n" + "fmla v15.4s, v31.4s, v8.4s\n" + "fmla v1.4s, v31.4s, v7.4s\n" + "fmla v21.4s, v31.4s, v9.4s\n" + "ldr s31, [x15, x27]\n" + "fmla v14.4s, v26.4s, v4.4s\n" + "ldr x15, [%[inptrs], 216]\n" + "fmla v10.4s, v26.4s, v5.4s\n" + "ldr s29, [x18, x27]\n" + "fmla v1.4s, v25.4s, v8.4s\n" + "ldr s28, [x24, x27]\n" + "fmla v13.4s, v31.4s, v3.4s\n" + "ldr x18, [%[inptrs], 176]\n" + "fmla v14.4s, v31.4s, v6.4s\n" + "ldr x24, [%[inptrs], 136]\n" + "fmla v12.4s, v31.4s, v4.4s\n" + "fmla v10.4s, v31.4s, v7.4s\n" + "fmla v11.4s, v31.4s, v5.4s\n" + "fmla v20.4s, v31.4s, v19.4s\n" + "fmla v0.4s, v29.4s, v3.4s\n" + "ldr s25, [x17, x27]\n" + "fmla v15.4s, v29.4s, v4.4s\n" + "fmla v21.4s, v29.4s, v5.4s\n" + "fmla v12.4s, v29.4s, v6.4s\n" + "fmla v10.4s, v29.4s, v8.4s\n" + "fmla v11.4s, v29.4s, v7.4s\n" + "fmla v20.4s, v29.4s, v9.4s\n" + "fmla v24.4s, v29.4s, v19.4s\n" + "fmla v23.4s, v29.4s, v22.4s\n" + "fmla v17.4s, v28.4s, v3.4s\n" + "ldr s29, [x16, x27]\n" + "fmla v15.4s, v28.4s, v6.4s\n" + "ldr s22, [x15, x27]\n" + "fmla v1.4s, v28.4s, v4.4s\n" + "ldr x16, [%[inptrs], 264]\n" + "fmla v11.4s, v28.4s, v8.4s\n" + "ldr x15, [%[inptrs], 224]\n" + "fmla v21.4s, v28.4s, v7.4s\n" + "fmla v24.4s, v28.4s, v9.4s\n" + "fmla v14.4s, v29.4s, v3.4s\n" + "ldr s27, [x18, x27]\n" + "fmla v1.4s, v25.4s, v6.4s\n" + "ldr x18, [%[inptrs], 184]\n" + "fmla v10.4s, v29.4s, v4.4s\n" + "fmla v20.4s, v29.4s, v5.4s\n" + "fmla v21.4s, v25.4s, v8.4s\n" + "ldr s26, [x24, x27]\n" + "fmla v12.4s, v22.4s, v3.4s\n" + "ldr s25, [x16, x27]\n" + "fmla v11.4s, v22.4s, v4.4s\n" + "ldr x16, [%[inptrs], 272]\n" + "fmla v10.4s, v22.4s, v6.4s\n" + "fmla v20.4s, v22.4s, v7.4s\n" + "fmla v24.4s, v22.4s, v5.4s\n" + "fmla v23.4s, v22.4s, v19.4s\n" + "fmla v15.4s, v27.4s, v3.4s\n" + "ldr s31, [x15, x27]\n" + "fmla v11.4s, v27.4s, v6.4s\n" + "ldr s22, [x18, x27]\n" + "fmla v21.4s, v27.4s, v4.4s\n" + "ldr x15, [%[inptrs], 232]\n" + "fmla v20.4s, v27.4s, v8.4s\n" + "fmla v24.4s, v27.4s, v7.4s\n" + "fmla v23.4s, v27.4s, v9.4s\n" + "ldr s19, [x16, x27]\n" + "fmla v1.4s, v26.4s, v3.4s\n" + "ldr s28, [x15, x27]\n" + "fmla v21.4s, v26.4s, v6.4s\n" + "ldr x16, [%[inptrs], 280]\n" + "fmla v24.4s, v26.4s, v8.4s\n" + "fmla v10.4s, v25.4s, v3.4s\n" + "fmla v20.4s, v25.4s, v4.4s\n" + "ldr s30, [x16, x27]\n" + "fmla v23.4s, v25.4s, v5.4s\n" + "add x27, x27, #4\n" + "fmla v11.4s, v31.4s, v3.4s\n" + "fmla v21.4s, v22.4s, v3.4s\n" + "fmla v24.4s, v31.4s, v4.4s\n" + "movi v29.16b, #0\n" + "fmla v20.4s, v31.4s, v6.4s\n" + "fmla v23.4s, v31.4s, v7.4s\n" + "fmax v2.4s, v2.4s, v29.4s\n" + "fmax v18.4s, v18.4s, v29.4s\n" + "fmla v24.4s, v22.4s, v6.4s\n" + "fmax v17.4s, v17.4s, v29.4s\n" + "fmla v20.4s, v19.4s, v3.4s\n" + "fmax v1.4s, v1.4s, v29.4s\n" + "str s2, [x20, x28]\n" + "fmla v23.4s, v22.4s, v8.4s\n" + "fmax v16.4s, v16.4s, v29.4s\n" + "ldr x20, [%[outptrs], 8]\n" + "fmla v24.4s, v28.4s, v3.4s\n" + "fmax v0.4s, v0.4s, v29.4s\n" + "str s18, [x20, x28]\n" + "fmax v15.4s, v15.4s, v29.4s\n" + "str s16, [x21, x28]\n" + "fmla v23.4s, v19.4s, v4.4s\n" + "fmax v21.4s, v21.4s, v29.4s\n" + "ldr x20, [%[outptrs], 16]\n" + "fmax v13.4s, v13.4s, v29.4s\n" + "ldr x21, [%[outptrs], 40]\n" + "str s17, [x20, x28]\n" + "fmax v12.4s, v12.4s, v29.4s\n" + "str s0, [x21, x28]\n" + "fmla v23.4s, v28.4s, v6.4s\n" + "str s13, [x22, x28]\n" + "fmax v11.4s, v11.4s, v29.4s\n" + "fmax v24.4s, v24.4s, v29.4s\n" + "ldr x20, [%[outptrs], 24]\n" + "fmax v14.4s, v14.4s, v29.4s\n" + "ldr x21, [%[outptrs], 48]\n" + "str s1, [x20, x28]\n" + "fmla v23.4s, v30.4s, v3.4s\n" + "str s15, [x21, x28]\n" + "fmax v10.4s, v10.4s, v29.4s\n" + "str s14, [x23, x28]\n" + "fmax v20.4s, v20.4s, v29.4s\n" + "ldr x21, [%[outptrs], 56]\n" + "ldr x22, [%[outptrs], 72]\n" + "ldr x23, [%[outptrs], 104]\n" + "fmax v23.4s, v23.4s, v29.4s\n" + "str s21, [x21, x28]\n" + "str s12, [x22, x28]\n" + "str s10, [x23, x28]\n" + "ldr x22, [%[outptrs], 80]\n" + "ldr x23, [%[outptrs], 112]\n" + "str s11, [x22, x28]\n" + "str s20, [x23, x28]\n" + "ldr x22, [%[outptrs], 88]\n" + "ldr x23, [%[outptrs], 120]\n" + "str s24, [x22, x28]\n" + "str s23, [x23, x28]\n" + "add x28, x28, #4\n" + "7:\n" + : [wbptr] "+r" (weight_bias_ptr) + : [n_channels] "r" ((long) n_channels), [outptrs] "r" (outptrs), [inptrs] "r" (inptrs) + : "cc", "v0", "v1", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v2", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v3", "v30", "v31", "v4", "v5", "v6", "v7", "v8", "v9", "x15", "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "memory" + ); +} + +template <> +template <> void Conv::execute_tile<ActivationFunction::ReLU6>( int n_channels, const void *weight_bias_ptr, |